# ====================================================== # robots.txt # Website: https://www.kalyantoursudaipur.com # Business: Kalyan Tours Udaipur # Updated: July 2026 # Purpose: # - Allow all major search engines # - Allow AI search / AEO / GEO retrieval bots # - Allow AI training bots # - Throttle/limit low-value SEO scraper bots # - Block only private/system folders # ====================================================== User-agent: * Allow: / # Block only private/system folders Disallow: /cgi-bin/ Disallow: /blog/wp-admin/ Allow: /blog/wp-admin/admin-ajax.php Disallow: /blog/wp-includes/ Disallow: /trackback/ Disallow: /comments/feed/ # ====================================================== # OpenAI # ====================================================== User-agent: GPTBot Allow: / User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / # ====================================================== # Anthropic # ====================================================== User-agent: ClaudeBot Allow: / User-agent: Claude-SearchBot Allow: / User-agent: Claude-User Allow: / User-agent: anthropic-ai Allow: / # ====================================================== # Google AI / Search # ====================================================== User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-News Allow: / User-agent: Googlebot-Video Allow: / User-agent: Google-Extended Allow: / # ====================================================== # Microsoft / Bing AI # ====================================================== User-agent: Bingbot Allow: / User-agent: MicrosoftPreview Allow: / User-agent: AdIdxBot Allow: / # ====================================================== # Perplexity AI # ====================================================== User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / # ====================================================== # You.com # ====================================================== User-agent: YouBot Allow: / # ====================================================== # Cohere # ====================================================== User-agent: cohere-ai Allow: / User-agent: cohere-training-data-crawler Allow: / # ====================================================== # Apple Intelligence # ====================================================== User-agent: Applebot Allow: / User-agent: Applebot-Extended Allow: / # ====================================================== # Meta AI # ====================================================== User-agent: FacebookBot Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: meta-externalfetcher Allow: / # ====================================================== # ByteDance / TikTok # ====================================================== User-agent: Bytespider Allow: / # ====================================================== # Amazon # ====================================================== User-agent: Amazonbot Allow: / # ====================================================== # Common Crawl (feeds many AI training sets) # ====================================================== User-agent: CCBot Allow: / # ====================================================== # Diffbot / Data extraction AI # ====================================================== User-agent: Diffbot Allow: / # ====================================================== # ImagesiftBot # ====================================================== User-agent: ImagesiftBot Allow: / # ====================================================== # Webz.io / Omgili # ====================================================== User-agent: Omgilibot Allow: / User-agent: Omgili Allow: / # ====================================================== # Timpi # ====================================================== User-agent: Timpibot Allow: / # ====================================================== # Huawei PetalBot # ====================================================== User-agent: PetalBot Allow: / # ====================================================== # Yahoo # ====================================================== User-agent: Slurp Allow: / # ====================================================== # Yandex # ====================================================== User-agent: Yandex Allow: / # ====================================================== # Baidu # ====================================================== User-agent: Baiduspider Allow: / # ====================================================== # DuckDuckGo # ====================================================== User-agent: DuckDuckBot Allow: / # ====================================================== # Brave Search # ====================================================== User-agent: Bravebot Allow: / # ====================================================== # Mojeek # ====================================================== User-agent: MojeekBot Allow: / # ====================================================== # Qwant # ====================================================== User-agent: Qwantify Allow: / # ====================================================== # SEO scraper bots — throttled, not blocked # (These don't help AEO/GEO visibility; they mainly # add server load. Crawl-delay keeps them from # hammering the server without fully blocking access.) # ====================================================== User-agent: AhrefsBot Allow: / User-agent: SemrushBot Allow: / User-agent: MJ12bot Allow: / User-agent: DotBot Allow: / # AI Discovery File #LLMS: https://www.kalyantoursudaipur.com/llms.txt # XML Sitemaps Sitemap: https://www.kalyantoursudaipur.com/sitemap.xml