# Robots.txt for ClientsNow.in
# Optimized for AI Crawlers and Answer Engine Optimization
# Last Updated: January 28, 2026

# ========================================
# DEFAULT CRAWLER ACCESS
# ========================================

User-agent: *
Allow: /
Crawl-delay: 1

# ========================================
# SITEMAP LOCATIONS
# ========================================

Sitemap: https://www.clientsnow.in/sitemap.xml

# ========================================
# AI CRAWLERS - EXPLICIT PERMISSIONS
# ========================================

# OpenAI GPTBot (ChatGPT)
User-agent: GPTBot
Allow: /
Allow: /seo-agency-in-ahmedabad.php
Allow: /website-development-agency.php
Allow: /digital-marketing-agency.php
Allow: /blog/
Allow: /about.php
Allow: /contact.php
Disallow: /admin/
Disallow: /private/
Disallow: /cart/
Disallow: /checkout/
Crawl-delay: 1

# ChatGPT User Agent
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

# Anthropic Claude AI
User-agent: anthropic-ai
Allow: /
Allow: /seo-agency-in-ahmedabad.php
Allow: /website-development-agency.php
Allow: /digital-marketing-agency.php
Allow: /blog/
Allow: /about.php
Allow: /contact.php
Disallow: /admin/
Disallow: /private/
Crawl-delay: 1

User-agent: ClaudeBot
Allow: /
Crawl-delay: 1

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /seo-agency-in-ahmedabad.php
Allow: /website-development-agency.php
Allow: /digital-marketing-agency.php
Allow: /blog/
Disallow: /admin/
Disallow: /private/
Crawl-delay: 1

# Google Extended (Bard/Gemini)
User-agent: Google-Extended
Allow: /
Crawl-delay: 0.5

# Common Crawl (Used by many AI models)
User-agent: CCBot
Allow: /
Crawl-delay: 2

# Apple Intelligence (Applebot)
User-agent: Applebot
Allow: /
Crawl-delay: 1

# Applebot Extended
User-agent: Applebot-Extended
Allow: /
Crawl-delay: 1

# Meta AI (Facebook)
User-agent: FacebookBot
Allow: /
Allow: /blog/
Crawl-delay: 1

User-agent: meta-externalagent
Allow: /
Crawl-delay: 1

# Cohere AI
User-agent: cohere-ai
Allow: /
Crawl-delay: 1

# Diffbot
User-agent: Diffbot
Allow: /
Crawl-delay: 2

# ========================================
# STANDARD SEARCH ENGINE BOTS
# ========================================

# Google
User-agent: Googlebot
Allow: /
Crawl-delay: 0.5

User-agent: Googlebot-Image
Allow: /assets/images/
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.png
Allow: /*.webp
Allow: /*.gif

User-agent: Googlebot-Video
Allow: /

# Bing (Microsoft)
User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: msnbot
Allow: /
Crawl-delay: 1

User-agent: BingPreview
Allow: /

# Yahoo
User-agent: Slurp
Allow: /
Crawl-delay: 1

# Yandex
User-agent: YandexBot
Allow: /
Crawl-delay: 2

# Baidu
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# ========================================
# DIRECTORIES TO ALLOW (HIGH PRIORITY)
# ========================================

Allow: /seo-agency-in-ahmedabad.php
Allow: /website-development-agency.php
Allow: /digital-marketing-agency.php
Allow: /local-seo-services-in-ahmedabad.php
Allow: /technical-seo-in-ahmedabad.php
Allow: /ecommerce-seo-services-in-ahmedabad.php
Allow: /aeo-services.php
Allow: /about.php
Allow: /contact.php
Allow: /blog/
Allow: /portfolio.php
Allow: /testimonials.php

# Allow important assets
Allow: /assets/css/
Allow: /assets/js/
Allow: /assets/images/

# ========================================
# DIRECTORIES TO DISALLOW (ALL BOTS)
# ========================================

# Admin and private areas
Disallow: /admin/
Disallow: /administrator/
Disallow: /wp-admin/
Disallow: /private/
Disallow: /secure/
Disallow: /member/
Disallow: /login/
Disallow: /dashboard/

# E-commerce areas (if applicable)
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /order/

# Technical directories
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /temp/
Disallow: /cache/
Disallow: /backup/

# Development and testing
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /beta/

# System files
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.zip$
Disallow: /*.tar$
Disallow: /*.gz$

# Duplicate content prevention
Disallow: /print/
Disallow: /search

# CMS specific (WordPress example)
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/
Disallow: /feed/
Disallow: /comments/
Disallow: /trackback/
Disallow: /xmlrpc.php

# ========================================
# SPECIFIC FILE TYPES TO BLOCK
# ========================================

# Block access to specific file types
Allow: /*.php
Disallow: /*.inc$
Disallow: /*.class$
Disallow: /*.json$

# ========================================
# BAD BOTS TO BLOCK
# ========================================

# Aggressive crawlers
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SeznamBot
Crawl-delay: 10

# Scrapers and bad bots
User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: CopyRightCheck
Disallow: /

User-agent: Crescent
Disallow: /

User-agent: Harvest
Disallow: /

User-agent: WBSearchBot
Disallow: /

# ========================================
# RATE LIMITING FOR AGGRESSIVE CRAWLERS
# ========================================

# If you want to allow but slow down certain crawlers
User-agent: PetalBot
Crawl-delay: 10

User-agent: Bytespider
Crawl-delay: 10

# ========================================
# OPTIONAL: BLOCK AI TRAINING (IF NEEDED)
# ========================================

# Uncomment these lines if you want to block AI bots
# from using your content for training

# User-agent: GPTBot
# Disallow: /

# User-agent: ChatGPT-User
# Disallow: /

# User-agent: anthropic-ai
# Disallow: /

# User-agent: ClaudeBot
# Disallow: /

# User-agent: CCBot
# Disallow: /

# User-agent: Google-Extended
# Disallow: /

# User-agent: PerplexityBot
# Disallow: /

# ========================================
# NOTES FOR IMPLEMENTATION
# ========================================

# 1. Upload this file to your website root directory
# 2. Test at: https://www.google.com/webmasters/tools/robots-testing-tool
# 3. Verify in Google Search Console
# 4. Monitor crawler activity in server logs
# 5. Update sitemap URLs if they differ
# 6. Adjust crawl-delay values based on server capacity
# 7. Review and update quarterly

# ========================================
# VALIDATION
# ========================================

# Validate this robots.txt at:
# - https://support.google.com/webmasters/answer/6062598
# - https://www.bing.com/webmasters/
# - Any robots.txt validator tool

# ========================================
# SUPPORT
# ========================================

# For questions about this robots.txt configuration:
# Contact: Clients Now SEO Agency
# Email: info@clientsnow.in
# Phone: +91-9924982323
# Website: https://www.clientsnow.in/

# Last Review Date: January 28, 2026
# Next Review Date: April 28, 2026