# robots.txt # ------------------------------------------------------------ # Block known aggressive / unwanted crawlers (as requested) # NOTE: robots.txt only works for bots that FOLLOW robots rules. # For "empty User-Agent" requests, robots.txt cannot reliably match. # Block empty User-Agent at server/WAF level (e.g., .htaccess / firewall). # ------------------------------------------------------------ User-agent: dotbot User-agent: DotBot User-agent: MJ12bot User-agent: Dalvik User-agent: Dalvik/2.1.0 (Linux; U; Android 9; SM-G955F Build/PPR1.180610.011) User-agent: Dalvik/2.1.0 User-agent: bidswitchbot/1.0 User-agent: BLEXBot User-agent: YandexBot User-agent: YandexBot/3.0 User-agent: SeznamBot User-agent: Mail.RU Bot User-agent: YandexMobileBot User-agent: YisouSpider User-agent: GrapeshotCrawler User-agent: GrapeshotCrawler/2.0 User-agent: SurdotlyBot User-agent: MojeekBot User-agent: Sogou web spider User-agent: ZoominfoBot User-agent: coccocbot-web/1.0 User-agent: Trident User-agent: proximic User-agent: LWP::Simple/6.22 User-agent: UptimeRobot/2.0 User-agent: Baiduspider Disallow: / # ------------------------------------------------------------ # Default rules (kept exactly as provided) # ------------------------------------------------------------ User-agent: * Allow: / Disallow: /admin/ Disallow: /checkout/ Disallow: /components/ Disallow: /includes/ Disallow: /log/ Disallow: /logs/ Disallow: /sitemaps/ Disallow: /sms/ Disallow: /user/ Disallow: /vendor/ Disallow: /whatsapp/ Disallow: /api/ # Explicit files (both pretty URL and .php variants where relevant) Disallow: /search.php Disallow: /search_suggest.php Disallow: /verify_payment.php Disallow: /thank-you.php Disallow: /cart Disallow: /cart.php Disallow: /checkout.php Disallow: /create_order.php Disallow: /insert_address.php Disallow: /get_payment_methods.php Disallow: /fetch_addresses.php Disallow: /delete_address.php Disallow: /razorpay_webhook.php Disallow: /db.php # ChatGPT Search / Shopping crawler User-agent: OAI-SearchBot Disallow: # When a user explicitly opens your site from ChatGPT User-agent: ChatGPT-User Disallow: