# robots.txt for forkful.cloud # No blank lines inside a group: older parsers treat a blank line as the # end of the group and drop the rules after it. # Every crawler, including search engines and AI assistants / AI search # (ChatGPT-User, OAI-SearchBot, Claude-User, Claude-SearchBot, # PerplexityBot, ...): everything except sign-in and account pages # (all account_*.php, including the old per-recipe # account_login.php?next=/id/ links). # Content-Signal (Cloudflare's content signals): search and AI answers yes, # AI training no. User-agent: * Content-Signal: search=yes, ai-input=yes, ai-train=no Crawl-delay: 1 Disallow: /account_ Disallow: /login.php Disallow: /logout.php # A redirect to a random recipe: nothing of its own, and a different page each time. Disallow: /random.php # Crawlers that collect content for training AI models: nothing. # (Google-Extended and Applebot-Extended only control AI training use; # blocking them doesn't affect Google or Apple search.) User-agent: Amazonbot User-agent: AI2Bot User-agent: Ai2Bot-Dolma User-agent: anthropic-ai User-agent: Applebot-Extended User-agent: Bytespider User-agent: CCBot User-agent: ClaudeBot User-agent: cohere-ai User-agent: cohere-training-data-crawler User-agent: CrawlFuseBench User-agent: ElevenlabsCrawler User-agent: ExaSearchBot User-agent: ExaSearchBot-Images User-agent: Diffbot User-agent: FacebookBot User-agent: Google-Extended User-agent: GPTBot User-agent: ImagesiftBot User-agent: img2dataset User-agent: meta-externalagent User-agent: omgili User-agent: omgilibot User-agent: PanguBot User-agent: Reflectionbot User-agent: ShapBot User-agent: SemrushBot User-agent: Timpibot User-agent: Webzio-Extended Disallow: / # SEO data tools that bring no visitors (AhrefsBot feeds the Yep search engine, but has brought no visitors, ever.) User-agent: AhrefsBot User-agent: BLEXBot User-agent: Barkrowler User-agent: DataForSeoBot User-agent: DotBot User-agent: MJ12bot User-agent: serpstatbot Disallow: / Sitemap: https://forkful.cloud/sitemap.xml