# ==============================================================================
# Pashmina Tours Robots.txt File
# ==============================================================================

# ------------------------------------------------------------------------------
# 1. ALLOWED SEARCH ENGINES & SEARCH-ONLY AI AGENTS (For Traffic and Citations)
# ------------------------------------------------------------------------------

# Allow standard search engines (Google, Bing, etc.)
User-agent: Googlebot
Disallow: /admin/
Disallow: /api/
Disallow: /checkout/
Allow: /

User-agent: Bingbot
Disallow: /admin/
Disallow: /api/
Disallow: /checkout/
Allow: /

# Allow ChatGPT Search (OAI-SearchBot) and GPTBot to crawl for real-time search recommendations and links
User-agent: OAI-SearchBot
Disallow: /admin/
Disallow: /api/
Disallow: /checkout/
Allow: /

User-agent: GPTBot
Disallow: /admin/
Disallow: /api/
Disallow: /checkout/
Allow: /

# Allow Perplexity AI to crawl for recommendations (AEO)
User-agent: PerplexityBot
Disallow: /admin/
Disallow: /api/
Disallow: /checkout/
Allow: /

# ------------------------------------------------------------------------------
# 2. DISALLOWED AI CRAWLERS (Blocks massive data-scraping for general LLM training)
# Note: GPTBot and PerplexityBot are ALLOWED in section 1 for AEO (AI Engine Optimization)
# ------------------------------------------------------------------------------

# Block Anthropic ClaudeBot (General training crawler)
User-agent: ClaudeBot
Disallow: /

# Allow Google-Extended (Gemini) to crawl for AI Search recommendations
User-agent: Google-Extended
Disallow: /admin/
Disallow: /api/
Disallow: /checkout/
Allow: /

# Block Common Crawl (Massive public web scraper)
User-agent: CCBot
Disallow: /

# Block other common AI training/crawling bots
User-agent: CohereBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: FacebookBot
Disallow: /

# ------------------------------------------------------------------------------
# 3. DISALLOWED SCAMP BOTS & AGGRESSIVE SCRAPERS
# ------------------------------------------------------------------------------
User-agent: Rogerbot
Disallow: /

User-agent: Dotbot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

# ------------------------------------------------------------------------------
# 4. DEFAULT RULES FOR ALL OTHER VISITORS
# ------------------------------------------------------------------------------
User-agent: *
Disallow: /admin/
Disallow: /api/
Disallow: /checkout/
Allow: /

# Sitemap Location (Change to your official domain once live)
Sitemap: https://www.pashminatours.com/sitemap.xml