# Robots.txt updated 30/04/2026 # Block AI agents and training crawlers # Allow search engines and other legitimate crawlers User-agent: GPTBot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: ChatGPT-User Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: ChatGPT Agent Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: OAI-SearchBot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Google-Extended Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: anthropic-ai Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: anthropic-ai/1.0 Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Claude-Web Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: ClaudeBot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Claude-User Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Claude-SearchBot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Claude-Web Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: PerplexityBot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Applebot-Extended Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Amazonbot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Amzn-SearchBot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Amzn-User Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: FacebookBot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Bytespider Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: CCBot Disallow: / User-agent: YouBot Disallow: / User-agent: Diffbot Disallow: / User-agent: ApifyBot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: ApifyWebsiteContentCrawler Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: AzureAI-SearchBot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Cloudflare-AutoRAG Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: CloudVertexBot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: DeepSeekBot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: FirecrawlAgent Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: FriendlyCrawler Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Google-Agent Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Google-CloudVertexBot Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Google-Firebase Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: KunatoCrawler Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: Scrapy Disallow: /blog/ Disallow: /the-paper/ Disallow: /podcasts-and-videos/podcasts/ Disallow: /podcasts-and-videos/videos/ Disallow: /courses/ User-agent: * Disallow: /Media/ Disallow: /4d422a2b-60d3-457f-8754-e8a09ebe3fb1