{
  "license": "CC BY 4.0",
  "generated": "2026-11-22T20:45:24.418Z",
  "source": "https://whoreadsthe.com",
  "agents": [
    {
      "id": "gptbot",
      "name": "GPTBot",
      "vendor": "openai",
      "purpose": "training",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/gptbot",
      "reference": "4EX-GRU-WZ8",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": "GPTBot",
        "uaExample": {
          "value": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot",
          "source": "https://developers.openai.com/api/docs/bots",
          "checked": "2026-11-22",
          "quote": "Example user-agent string (the version number may change): Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot"
        },
        "robotsPolicy": {
          "value": "honours",
          "source": "https://developers.openai.com/api/docs/bots",
          "checked": "2026-11-22",
          "quote": "Disallowing GPTBot indicates a site’s content should not be used in training generative AI foundation models."
        },
        "runsJs": null,
        "ipList": {
          "value": "https://openai.com/gptbot.json",
          "source": "https://developers.openai.com/api/docs/bots",
          "checked": "2026-11-22",
          "quote": "Published IP addresses: https://openai.com/gptbot.json"
        },
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": "2026-11-22",
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": []
    },
    {
      "id": "oai-searchbot",
      "name": "OAI-SearchBot",
      "vendor": "openai",
      "purpose": "search",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/oai-searchbot",
      "reference": "6VH-8TY-QLJ",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": "OAI-SearchBot",
        "uaExample": {
          "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot",
          "source": "https://developers.openai.com/api/docs/bots",
          "checked": "2026-11-22",
          "quote": "Example user-agent string (the version number may change): Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36; compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot"
        },
        "robotsPolicy": {
          "value": "honours",
          "source": "https://developers.openai.com/api/docs/bots",
          "checked": "2026-11-22",
          "quote": "For search results, please note it can take ~24 hours from a site’s robots.txt update for our systems to adjust."
        },
        "runsJs": null,
        "ipList": {
          "value": "https://openai.com/searchbot.json",
          "source": "https://developers.openai.com/api/docs/bots",
          "checked": "2026-11-22",
          "quote": "Published IP addresses: https://openai.com/searchbot.json"
        },
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": "2026-11-22",
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": [
        {
          "date": "2026-11-22",
          "site": "probe",
          "summary": "OAI-SearchBot requested robots.txt 10 seconds after the first ChatGPT-User fetch in the prompted test. It fetched no page.",
          "source": "Who reads the web probe findings, 2026-11-22"
        }
      ]
    },
    {
      "id": "chatgpt-user",
      "name": "ChatGPT-User",
      "vendor": "openai",
      "purpose": "user",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/chatgpt-user",
      "reference": "55Q-N83-CVM",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": null,
        "uaExample": {
          "value": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
          "source": "https://developers.openai.com/api/docs/bots",
          "checked": "2026-11-22",
          "quote": "Full user-agent string: Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot"
        },
        "robotsPolicy": {
          "value": "may_ignore",
          "source": "https://developers.openai.com/api/docs/bots",
          "checked": "2026-11-22",
          "quote": "Because these actions are initiated by a user, robots.txt rules may not apply."
        },
        "runsJs": null,
        "ipList": {
          "value": "https://openai.com/chatgpt-user.json",
          "source": "https://developers.openai.com/api/docs/bots",
          "checked": "2026-11-22",
          "quote": "Published IP addresses: https://openai.com/chatgpt-user.json"
        },
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": "2026-11-22",
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": [
        {
          "date": "2026-11-22",
          "site": "probe",
          "summary": "We asked ChatGPT for the reference code on two unlinked, unindexed pages, one of them disallowed in robots.txt. ChatGPT-User fetched both within 5 seconds without requesting robots.txt, and both answers quoted the correct code.",
          "source": "Who reads the web probe findings, 2026-11-22"
        }
      ]
    },
    {
      "id": "chatgpt-agent",
      "name": "ChatGPT agent",
      "vendor": "openai",
      "purpose": "agentic",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/chatgpt-agent",
      "reference": "8LF-A9P-LVT",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": null,
        "uaExample": null,
        "robotsPolicy": null,
        "runsJs": null,
        "ipList": null,
        "signatureAgent": {
          "value": "https://chatgpt.com",
          "source": "https://developers.openai.com/commerce/agent-access",
          "checked": "2026-11-22",
          "quote": "A Signature-Agent header set to \"https://chatgpt.com\"."
        },
        "rdnsSuffixes": null,
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": []
    },
    {
      "id": "claudebot",
      "name": "ClaudeBot",
      "vendor": "anthropic",
      "purpose": "training",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/claudebot",
      "reference": "VNK-AWW-J8J",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": "ClaudeBot",
        "uaExample": null,
        "robotsPolicy": {
          "value": "honours",
          "source": "https://support.claude.com/en/articles/8896518",
          "checked": "2026-11-22",
          "quote": "Anthropic’s Bots respect “do not crawl” signals by honoring industry standard directives in robots.txt."
        },
        "runsJs": null,
        "ipList": {
          "value": "https://claude.com/crawling/bots.json",
          "source": "https://support.claude.com/en/articles/8896518",
          "checked": "2026-11-22",
          "quote": "If a crawler has a source IP address on this list, it indicates that the crawler is coming from Anthropic."
        },
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": {
          "value": {
            "cidrs": [
              "160.79.104.0/21"
            ]
          },
          "source": "https://rdap.arin.net/registry/ip/160.79.104.0/21",
          "checked": "2026-11-22"
        }
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "ipListNote": "The vendor publishes one IP list for all of its bots. The list does not say which addresses belong to this agent.",
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": []
    },
    {
      "id": "claude-searchbot",
      "name": "Claude-SearchBot",
      "vendor": "anthropic",
      "purpose": "search",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/claude-searchbot",
      "reference": "GHG-A9C-56P",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": null,
        "uaExample": null,
        "robotsPolicy": {
          "value": "honours",
          "source": "https://support.claude.com/en/articles/8896518",
          "checked": "2026-11-22",
          "quote": "Anthropic’s Bots respect “do not crawl” signals by honoring industry standard directives in robots.txt."
        },
        "runsJs": null,
        "ipList": {
          "value": "https://claude.com/crawling/bots.json",
          "source": "https://support.claude.com/en/articles/8896518",
          "checked": "2026-11-22",
          "quote": "If a crawler has a source IP address on this list, it indicates that the crawler is coming from Anthropic."
        },
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": {
          "value": {
            "cidrs": [
              "160.79.104.0/21"
            ]
          },
          "source": "https://rdap.arin.net/registry/ip/160.79.104.0/21",
          "checked": "2026-11-22"
        }
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "ipListNote": "The vendor publishes one IP list for all of its bots. The list does not say which addresses belong to this agent.",
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": []
    },
    {
      "id": "claude-user",
      "name": "Claude-User",
      "vendor": "anthropic",
      "purpose": "user",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/claude-user",
      "reference": "W67-HJM-87R",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": null,
        "uaExample": null,
        "robotsPolicy": {
          "value": "honours",
          "source": "https://support.claude.com/en/articles/8896518",
          "checked": "2026-11-22",
          "quote": "Anthropic’s Bots respect “do not crawl” signals by honoring industry standard directives in robots.txt."
        },
        "runsJs": null,
        "ipList": {
          "value": "https://claude.com/crawling/bots.json",
          "source": "https://support.claude.com/en/articles/8896518",
          "checked": "2026-11-22",
          "quote": "If a crawler has a source IP address on this list, it indicates that the crawler is coming from Anthropic."
        },
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": {
          "value": {
            "cidrs": [
              "160.79.104.0/21"
            ]
          },
          "source": "https://rdap.arin.net/registry/ip/160.79.104.0/21",
          "checked": "2026-11-22"
        }
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "ipListNote": "The vendor publishes one IP list for all of its bots. The list does not say which addresses belong to this agent.",
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": [
        {
          "date": "2026-11-22",
          "site": "probe",
          "summary": "We asked Claude (claude.ai) for the reference code on two unlinked, unindexed pages, one of them disallowed in robots.txt. Claude-User fetched robots.txt and the allowed page, and the answer quoted the correct code. It made no request for the disallowed page, and the answer said robots.txt blocks it.",
          "source": "Who reads the web probe findings, 2026-11-22"
        }
      ]
    },
    {
      "id": "googlebot",
      "name": "Googlebot",
      "vendor": "google",
      "purpose": "search",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/googlebot",
      "reference": "F94-ZR4-WPW",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": "Googlebot",
        "uaExample": {
          "value": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
          "checked": "2026-11-22"
        },
        "robotsPolicy": {
          "value": "honours",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
          "checked": "2026-11-22",
          "quote": "They always obey robots.txt rules when crawling automatically."
        },
        "runsJs": null,
        "ipList": {
          "value": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
          "source": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot",
          "checked": "2026-11-22",
          "quote": "Common crawlers | common-crawlers.json | crawl-***-***-***-***.googlebot.com or geo-crawl-***-***-***-***.geo.googlebot.com"
        },
        "signatureAgent": null,
        "rdnsSuffixes": {
          "value": [
            ".googlebot.com",
            ".geo.googlebot.com"
          ],
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
          "checked": "2026-11-22",
          "quote": "The common crawlers generally crawl from the IP ranges published in the common-crawlers.json object, and the reverse DNS mask of their hostname matches crawl-***-***-***-***.googlebot.com or geo-crawl-***-***-***-***.geo.googlebot.com."
        },
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": []
    },
    {
      "id": "google-extended",
      "name": "Google-Extended",
      "vendor": "google",
      "purpose": "training",
      "observable": false,
      "url": "https://whoreadsthe.com/agents/google-extended",
      "reference": "GNK-MDE-5ZJ",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": "Google-Extended",
        "uaExample": null,
        "robotsPolicy": null,
        "runsJs": null,
        "ipList": null,
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": null
      },
      "observed": null,
      "disagreements": [],
      "tests": []
    },
    {
      "id": "google-agent",
      "name": "Google-Agent",
      "vendor": "google",
      "purpose": "agentic",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/google-agent",
      "reference": "P8X-NY4-JMS",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": null,
        "uaExample": {
          "value": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent) Chrome/W.X.Y.Z Safari/537.36",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
          "checked": "2026-11-22",
          "quote": "Desktop agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Google-Agent; +https://developers.google.com/crawling/docs/crawlers-fetchers/google-agent) Chrome/W.X.Y.Z Safari/537.36"
        },
        "robotsPolicy": {
          "value": "may_ignore",
          "source": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers",
          "checked": "2026-11-22",
          "quote": "Because the fetch was requested by a user, these fetchers generally ignore robots.txt rules."
        },
        "runsJs": null,
        "ipList": {
          "value": "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json",
          "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/allow-google-agent",
          "checked": "2026-11-22",
          "quote": "Each Google-Agent request includes a User-Agent HTTP request header containing \"Google-Agent\" and comes from an IP address from user-triggered-agents.json."
        },
        "signatureAgent": {
          "value": "https://agent.bot.goog",
          "source": "https://developers.google.com/crawling/docs/crawlers-fetchers/allow-google-agent",
          "checked": "2026-11-22",
          "quote": "Participating requests include: A Signature-Agent header set to g=\"https://agent.bot.goog\""
        },
        "rdnsSuffixes": null,
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": []
    },
    {
      "id": "google-bare",
      "name": "Gemini user fetcher (bare \"Google\" user agent)",
      "vendor": "google",
      "purpose": "user",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/google-bare",
      "reference": "RYS-7BA-7VL",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": null,
        "uaExample": null,
        "robotsPolicy": null,
        "runsJs": null,
        "ipList": null,
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": {
          "value": {
            "asns": [
              15169
            ]
          },
          "source": "https://rdap.arin.net/registry/autnum/15169",
          "checked": "2026-11-22",
          "quote": "AS15169 is registered to Google LLC. It is Google's own production network; Google Cloud customers use other networks."
        }
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": [
        {
          "date": "2026-11-22",
          "site": "probe",
          "summary": "We asked Gemini for the reference code on two unlinked, unindexed pages, one of them disallowed in robots.txt. A fetcher whose whole user agent was the word \"Google\" fetched both about 20 seconds later without requesting robots.txt, and both answers quoted the correct code.",
          "source": "Who reads the web probe findings, 2026-11-22"
        }
      ]
    },
    {
      "id": "bingbot",
      "name": "Bingbot",
      "vendor": "microsoft",
      "purpose": "search",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/bingbot",
      "reference": "A77-4P6-XLG",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": null,
        "uaExample": null,
        "robotsPolicy": null,
        "runsJs": null,
        "ipList": {
          "value": "https://www.bing.com/toolbox/bingbot.json",
          "source": "https://www.bing.com/toolbox/bingbot.json",
          "checked": "2026-11-22"
        },
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": []
    },
    {
      "id": "copilot",
      "name": "Microsoft Copilot",
      "vendor": "microsoft",
      "purpose": "user",
      "observable": false,
      "url": "https://whoreadsthe.com/agents/copilot",
      "reference": "7CD-92C-Q3Y",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": null,
        "uaExample": null,
        "robotsPolicy": null,
        "runsJs": null,
        "ipList": null,
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": null
      },
      "observed": null,
      "disagreements": [],
      "tests": [
        {
          "date": "2026-11-22",
          "site": "probe",
          "summary": "We asked Microsoft Copilot for the reference code on two unlinked, unindexed pages. No request reached the site. Both answers said the page could not be retrieved.",
          "source": "Who reads the web probe findings, 2026-11-22"
        }
      ]
    },
    {
      "id": "perplexitybot",
      "name": "PerplexityBot",
      "vendor": "perplexity",
      "purpose": "search",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/perplexitybot",
      "reference": "5L9-W6U-EUG",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": "PerplexityBot",
        "uaExample": {
          "value": "Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
          "source": "https://docs.perplexity.ai/guides/bots",
          "checked": "2026-11-22",
          "quote": "Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)"
        },
        "robotsPolicy": null,
        "runsJs": null,
        "ipList": {
          "value": "https://www.perplexity.ai/perplexitybot.json",
          "source": "https://docs.perplexity.ai/guides/bots",
          "checked": "2026-11-22"
        },
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": []
    },
    {
      "id": "perplexity-user",
      "name": "Perplexity-User",
      "vendor": "perplexity",
      "purpose": "user",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/perplexity-user",
      "reference": "JBE-XMW-LE2",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": "Perplexity-User",
        "uaExample": {
          "value": "Mozilla/5.0 (compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)",
          "source": "https://docs.perplexity.ai/guides/bots",
          "checked": "2026-11-22",
          "quote": "Mozilla/5.0 (compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)"
        },
        "robotsPolicy": {
          "value": "may_ignore",
          "source": "https://docs.perplexity.ai/guides/bots",
          "checked": "2026-11-22",
          "quote": "Since a user requested the fetch, this fetcher generally ignores robots.txt rules."
        },
        "runsJs": null,
        "ipList": {
          "value": "https://www.perplexity.ai/perplexity-user.json",
          "source": "https://docs.perplexity.ai/guides/bots",
          "checked": "2026-11-22"
        },
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": [
        {
          "date": "2026-11-22",
          "site": "probe",
          "summary": "We asked Perplexity four times, in two rounds, for the reference code on two unlinked, unindexed pages. No request from Perplexity reached the site. The answers said the page could not be retrieved.",
          "source": "Who reads the web probe findings, 2026-11-22"
        }
      ]
    },
    {
      "id": "applebot",
      "name": "Applebot",
      "vendor": "apple",
      "purpose": "search",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/applebot",
      "reference": "BUU-D6Q-MKB",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": "Applebot",
        "uaExample": {
          "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)",
          "source": "https://support.apple.com/en-us/119829",
          "checked": "2026-11-22"
        },
        "robotsPolicy": {
          "value": "honours",
          "source": "https://support.apple.com/en-us/119829",
          "checked": "2026-11-22",
          "quote": "Applebot respects standard robots.txt directives in general search crawls that are targeted at Applebot."
        },
        "runsJs": null,
        "ipList": {
          "value": "https://search.developer.apple.com/applebot.json",
          "source": "https://support.apple.com/en-us/119829",
          "checked": "2026-11-22",
          "quote": "Another way is to match the IP address with a CIDR prefix contained in the following JSON file: Applebot IP CIDRs."
        },
        "signatureAgent": null,
        "rdnsSuffixes": {
          "value": [
            ".applebot.apple.com"
          ],
          "source": "https://support.apple.com/en-us/119829",
          "checked": "2026-11-22",
          "quote": "Traffic coming from Applebot is generally identified by using reverse DNS in the *.applebot.apple.com domain."
        },
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": []
    },
    {
      "id": "ccbot",
      "name": "CCBot",
      "vendor": "commoncrawl",
      "purpose": "training",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/ccbot",
      "reference": "H33-3XJ-SBP",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": "CCBot",
        "uaExample": {
          "value": "CCBot/2.0 (https://commoncrawl.org/faq/)",
          "source": "https://commoncrawl.org/ccbot",
          "checked": "2026-11-22",
          "quote": "CCBot/2.0 (https://commoncrawl.org/faq/)"
        },
        "robotsPolicy": {
          "value": "honours",
          "source": "https://commoncrawl.org/faq",
          "checked": "2026-11-22",
          "quote": "checking first the robots.txt, and if crawling a page is allowed, fetches pages using HTTP GET requests."
        },
        "runsJs": {
          "value": false,
          "source": "https://commoncrawl.org/faq",
          "checked": "2026-11-22",
          "quote": "Currently, JavaScript is not executed and Cookies are not used."
        },
        "ipList": {
          "value": "https://index.commoncrawl.org/ccbot.json",
          "source": "https://commoncrawl.org/ccbot",
          "checked": "2026-11-22",
          "quote": "These IP ranges (v4 and v6) are also provided as JSON at https://index.commoncrawl.org/ccbot.json."
        },
        "signatureAgent": null,
        "rdnsSuffixes": {
          "value": [
            ".crawl.commoncrawl.org"
          ],
          "source": "https://commoncrawl.org/ccbot",
          "checked": "2026-11-22",
          "quote": "CCBot is now run on dedicated IP address ranges with reverse DNS (example: 18.97.14.84 resolves to 18-97-14-84.crawl.commoncrawl.org.)"
        },
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": []
    },
    {
      "id": "meta-externalagent",
      "name": "Meta-ExternalAgent",
      "vendor": "meta",
      "purpose": "training",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/meta-externalagent",
      "reference": "TGT-7C9-8XT",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": "meta-externalagent",
        "uaExample": {
          "value": "meta-externalagent/1.1",
          "source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
          "checked": "2026-11-22",
          "quote": "meta-externalagent/1.1"
        },
        "robotsPolicy": null,
        "runsJs": null,
        "ipList": null,
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": []
    },
    {
      "id": "bytespider",
      "name": "Bytespider",
      "vendor": "bytedance",
      "purpose": "training",
      "observable": true,
      "url": "https://whoreadsthe.com/agents/bytespider",
      "reference": "GZK-8J8-LW5",
      "license": "CC BY 4.0",
      "documented": {
        "robotsToken": null,
        "uaExample": null,
        "robotsPolicy": null,
        "runsJs": null,
        "ipList": null,
        "signatureAgent": null,
        "rdnsSuffixes": null,
        "network": null
      },
      "observed": {
        "allTime": [],
        "last30": null,
        "ipListLoaded": null,
        "note": "Not observed yet"
      },
      "disagreements": [],
      "tests": []
    }
  ]
}
