# Learners Ink - robots.txt # Global accredited certification training (PMP, PRINCE2, ITIL, Scrum, CBAP, # Lean Six Sigma, CISSP, CEH, AWS, Azure, Data, AI, DevOps). # --------------------------------------------------------------- # Default: all crawlers welcome on public pages # --------------------------------------------------------------- User-agent: * Allow: / Disallow: /api/ Disallow: /lovable/ Disallow: /admin Disallow: /admin/ Disallow: /auth Disallow: /checkout Disallow: /checkout/ Disallow: /thank-you Disallow: /thank-you/ Disallow: /thank-you-enquiry Disallow: /checkout-failed/ Disallow: /*?*utm_ Disallow: /*?*gclid= Disallow: /*?*fbclid= Crawl-delay: 1 # --------------------------------------------------------------- # Major search engines # --------------------------------------------------------------- User-agent: Googlebot Allow: / Disallow: /api/ Disallow: /admin Disallow: /auth Disallow: /checkout User-agent: Googlebot-Image Allow: / User-agent: Googlebot-News Allow: /blog/ User-agent: Bingbot Allow: / Disallow: /api/ Disallow: /admin Disallow: /auth Disallow: /checkout User-agent: DuckDuckBot Allow: / User-agent: YandexBot Allow: / User-agent: Baiduspider Allow: / User-agent: Slurp Allow: / User-agent: Applebot Allow: / # --------------------------------------------------------------- # AI / LLM crawlers (AEO + GEO discoverability) # --------------------------------------------------------------- User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: OAI-SearchBot Allow: / User-agent: Google-Extended Allow: / User-agent: GoogleOther Allow: / User-agent: ClaudeBot Allow: / User-agent: Claude-Web Allow: / User-agent: anthropic-ai Allow: / User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / User-agent: Applebot-Extended Allow: / User-agent: CCBot Allow: / User-agent: cohere-ai Allow: / User-agent: Amazonbot Allow: / User-agent: Bytespider Allow: / User-agent: DuckAssistBot Allow: / User-agent: YouBot Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: FacebookBot Allow: / User-agent: Diffbot Allow: / User-agent: MistralAI-User Allow: / User-agent: Mistral-User Allow: / User-agent: MistralBot Allow: / User-agent: Timpibot Allow: / User-agent: Omgilibot Allow: / User-agent: ImagesiftBot Allow: / # --------------------------------------------------------------- # Additional AI / LLM crawlers explicitly allowed # (Google Gemini, xAI Grok, Alibaba Qwen, Mistral, DeepSeek, Kagi, # Neeva, Phind, You, Komo, Andi, Brave Leo, Poe, HuggingFace, etc.) # --------------------------------------------------------------- User-agent: Google-CloudVertexBot Allow: / User-agent: GeminiBot Allow: / User-agent: Google-Gemini Allow: / User-agent: Gemini-Deep-Research Allow: / User-agent: Grok Allow: / User-agent: Grokbot Allow: / User-agent: xAI-Bot Allow: / User-agent: xAI Allow: / User-agent: QwenBot Allow: / User-agent: Qwen-Bot Allow: / User-agent: AlibabaBot Allow: / User-agent: DeepSeekBot Allow: / User-agent: DeepSeek Allow: / User-agent: KagiBot Allow: / User-agent: PhindBot Allow: / User-agent: YouBot-AI Allow: / User-agent: KomoBot Allow: / User-agent: AndiBot Allow: / User-agent: BraveBot Allow: / User-agent: LeoBot Allow: / User-agent: PoeBot Allow: / User-agent: HuggingFaceBot Allow: / User-agent: NeevaBot Allow: / User-agent: WebPilot Allow: / User-agent: SciSpaceBot Allow: / User-agent: NotebookLM Allow: / User-agent: Meta-ExternalFetcher Allow: / User-agent: LinkedInBot Allow: / User-agent: TwitterBot Allow: / User-agent: Twitterbot Allow: / # --------------------------------------------------------------- # Block aggressive SEO scrapers # --------------------------------------------------------------- User-agent: AhrefsBot Crawl-delay: 10 User-agent: SemrushBot Crawl-delay: 10 User-agent: MJ12bot Crawl-delay: 10 User-agent: DotBot Crawl-delay: 10 # --------------------------------------------------------------- # Discoverability files # --------------------------------------------------------------- Sitemap: https://www.learnersink.com/sitemap.xml Sitemap: https://www.learnersink.com/sitemap-core.xml Sitemap: https://www.learnersink.com/sitemap-classroom.xml Sitemap: https://www.learnersink.com/sitemap-online.xml Sitemap: https://www.learnersink.com/sitemap-category-geo.xml Sitemap: https://www.learnersink.com/sitemap-hubs.xml Sitemap: https://www.learnersink.com/sitemap-blog.xml Sitemap: https://www.learnersink.com/sitemap-images.xml Sitemap: https://www.learnersink.com/sitemap.txt Sitemap: https://www.learnersink.com/sitemap1.txt Sitemap: https://www.learnersink.com/sitemap2.txt