# Robots.txt for Dr. Avi Weinberg - Plastic Surgery Website
# Last Updated: July 2025
# Website: https://dr-weinberg.com
# =================================================

# DEFAULT RULES - ALL SEARCH ENGINES
User-agent: *
# WordPress Core Protection
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /wp-config.php
Disallow: /wp-settings.php

# Search and Query Parameters
Disallow: /search/
Disallow: /*?s=
Disallow: /*&s=
Disallow: /*?p=*
Disallow: /*?attachment_id=*
Disallow: /*?replytocom=*
Disallow: /*?preview=*

# Feed URLs
Disallow: /feed/
Disallow: */feed/
Disallow: */comments/feed/
Disallow: /*/*/feed/

# Archive and Pagination
Disallow: /author/
Disallow: /tag/
Disallow: /page/*/
Disallow: /*?year=*
Disallow: /*?monthnum=*
Disallow: /*?day=*
Disallow: /*?hour=*
Disallow: /*?minute=*
Disallow: /*?second=*
Disallow: /*?w=*
Disallow: /*?m=*

# Allow Essential Resources (No wildcards in paths)
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: /wp-includes/
Allow: *.css$
Allow: *.js$
Allow: *.jpg$
Allow: *.jpeg$
Allow: *.png$
Allow: *.gif$
Allow: *.webp$
Allow: *.svg$
Allow: *.woff$
Allow: *.woff2$
Allow: *.ttf$
Allow: *.eot$

# Medical Content - Allow (Important for medical SEO)
Allow: /articles/
Allow: /surgeries/
Allow: /surgeries_cat/
# Hebrew content is allowed by default

# PDF Documents - Allow (Medical information)
Allow: *.pdf$

# Crawl Delay (for bots that support it)
Crawl-delay: 5

# =================================================
# GOOGLE SPECIFIC
# =================================================
User-agent: Googlebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
# Note: Google ignores crawl-delay directive

User-agent: Googlebot-Image
Allow: /
# Critical for before/after photos visibility

User-agent: Googlebot-Mobile
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: AdsBot-Google
Disallow:

User-agent: AdsBot-Google-Mobile
Disallow:

# =================================================
# OTHER MAJOR SEARCH ENGINES
# =================================================
User-agent: Bingbot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
# Note: Bing ignores crawl-delay directive

User-agent: Yandex
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 5

User-agent: Baiduspider
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 10

# =================================================
# AI SEARCH ENGINES - ALLOWED (Medical Visibility)
# =================================================
# OpenAI Search
User-agent: OAI-SearchBot
Allow: /

# ChatGPT Browsing (User-initiated, not training)
User-agent: ChatGPT-User
Allow: /

User-agent: ChatGPT-User/2.0
Allow: /

# Perplexity AI Search
User-agent: PerplexityBot
Allow: /

# You.com Search
User-agent: YouBot
Allow: /

# Additional AI Search Engines
User-agent: NeevaBot
Allow: /

User-agent: Phind
Allow: /

User-agent: KomoBot
Allow: /

User-agent: Quora-Result-Bot
Allow: /

# =================================================
# AI TRAINING BOTS - BLOCKED
# =================================================
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: facebookexternalhit
Crawl-delay: 30

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: PetalBot
Disallow: /

# =================================================
# SEO TOOLS & SCRAPERS - BLOCKED
# =================================================
User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Megaindex.ru
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: GetintentCrawler
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: BacklinkCrawler
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Moz
Disallow: /

User-agent: Rogerbot
Disallow: /

User-agent: ZoomBot
Disallow: /

User-agent: SEOlyticsCrawler
Disallow: /

# =================================================
# SITEMAPS
# =================================================
Sitemap: https://dr-weinberg.com/sitemap_index.xml

# =================================================
# IMPLEMENTATION NOTES
# =================================================
# 1. WordPress structure protection with rendering support
# 2. All content is indexable (per site analysis)
# 3. Special allowance for medical content and images
# 4. Hebrew URLs supported (allowed by default)
# 5. Before/after photos allowed (by default)
# 6. AI search visibility for medical queries
# 7. AI training data collection blocked
# 8. Lower crawl-delay (5) for smaller site
# 9. No wildcards in path segments
# 10. Font files explicitly allowed
#
# Based on July 2025 analysis: 306/306 pages indexable
# No non-indexable pages identified
# =================================================