{
  "$comment": "robots.txt tokens for AI crawlers, each row from the vendor's own page. Served at /open-source/answer-crawler-check/crawler-tokens.json and shown at /open-source/answer-crawler-check/crawler-tokens/. Re-read every source before changing a row, and move `checked` with it.",
  "checked": "2026-11-08",
  "sources": {
    "anthropic": {
      "title": "Does Anthropic crawl data from the web, and how can site owners block the crawler?",
      "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
    },
    "openai": {
      "title": "Overview of OpenAI Crawlers",
      "url": "https://developers.openai.com/api/docs/bots"
    },
    "google": {
      "title": "Google's common crawlers",
      "url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers"
    },
    "perplexity": {
      "title": "Perplexity Crawlers",
      "url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers"
    }
  },
  "replace": [
    {
      "token": "anthropic-ai",
      "vendor": "anthropic",
      "status": "Not on Anthropic's crawler page, which names three bots",
      "use": [
        "ClaudeBot",
        "Claude-SearchBot",
        "Claude-User"
      ]
    },
    {
      "token": "Claude-Web",
      "vendor": "anthropic",
      "status": "Not on Anthropic's crawler page, which names three bots",
      "use": [
        "ClaudeBot",
        "Claude-SearchBot",
        "Claude-User"
      ]
    }
  ],
  "current": [
    {
      "token": "ClaudeBot",
      "vendor": "anthropic",
      "role": "training",
      "controls": "Collects web content that could contribute to training Anthropic's models.",
      "blocking": "Anthropic says it signals that the site's future materials should be excluded from its training datasets."
    },
    {
      "token": "Claude-SearchBot",
      "vendor": "anthropic",
      "role": "search",
      "controls": "Crawls to improve the quality of Claude's search results.",
      "blocking": "Anthropic says it stops your content being indexed for search, which can reduce your visibility in Claude's search results."
    },
    {
      "token": "Claude-User",
      "vendor": "anthropic",
      "role": "user",
      "controls": "Fetches pages when a person asks Claude a question.",
      "blocking": "Anthropic says Claude can no longer retrieve your pages in answer to a user's query."
    },
    {
      "token": "GPTBot",
      "vendor": "openai",
      "role": "training",
      "controls": "Crawls content that may be used to train OpenAI's foundation models.",
      "blocking": "OpenAI says it indicates the content should not be used for training. Each OpenAI setting is independent of the others."
    },
    {
      "token": "OAI-SearchBot",
      "vendor": "openai",
      "role": "search",
      "controls": "Surfaces websites in ChatGPT's search features.",
      "blocking": "OpenAI says opted-out sites are not shown in ChatGPT search answers, though they can still appear as navigational links."
    },
    {
      "token": "ChatGPT-User",
      "vendor": "openai",
      "role": "user",
      "controls": "Visits pages for actions a user takes in ChatGPT and Custom GPTs.",
      "blocking": "OpenAI says robots.txt rules may not apply to it, because a user started the request. It is not used to decide whether content appears in search."
    },
    {
      "token": "OAI-AdsBot",
      "vendor": "openai",
      "role": "other",
      "controls": "Visits only landing pages submitted as ads on ChatGPT, to check them against OpenAI's policies.",
      "blocking": "OpenAI says its data is not used to train foundation models."
    },
    {
      "token": "Google-Extended",
      "vendor": "google",
      "role": "training",
      "controls": "Whether content Google crawls may be used to train future Gemini models and for grounding in Gemini Apps and Vertex AI.",
      "blocking": "Google says it has no effect on inclusion in Google Search and is not a ranking signal. It has no user agent string of its own. Google crawls with its existing user agents."
    },
    {
      "token": "PerplexityBot",
      "vendor": "perplexity",
      "role": "search",
      "controls": "Surfaces and links websites in Perplexity's search results. Perplexity says it is not used to crawl content for AI foundation models.",
      "blocking": "Perplexity recommends allowing it for your site to appear in its search results."
    },
    {
      "token": "Perplexity-User",
      "vendor": "perplexity",
      "role": "user",
      "controls": "Visits a page when a user asks Perplexity a question, and can link it in the answer.",
      "blocking": "Perplexity says this fetcher generally ignores robots.txt rules, because a user requested the fetch."
    }
  ]
}