{
 "scanned": "2026-10-30",
 "vantage": "TestedActually build machine, GET / with the crawler's exact user-agent, redirects followed",
 "method": [
  "robots: fetch /robots.txt as ourselves, parse the User-agent group naming the bot; block-all = Disallow: / in that group, restricted = other rules, no-rule = no group names the bot",
  "http: fetch / with the bot's exact user-agent; record status plus Cloudflare challenge markers (CF-Mitigated: challenge)"
 ],
 "vendors": [
  "openai.com",
  "anthropic.com",
  "google.com",
  "perplexity.ai",
  "mistral.ai",
  "cohere.com",
  "meta.ai",
  "x.ai"
 ],
 "crawlers": [
  "GPTBot",
  "OAI-SearchBot",
  "ChatGPT-User",
  "ClaudeBot",
  "Claude-SearchBot",
  "Google-Extended",
  "PerplexityBot",
  "CCBot"
 ],
 "matrix": {
  "openai.com": {
   "GPTBot": {
    "robots": "no-rule",
    "http": "403 (challenge)"
   },
   "OAI-SearchBot": {
    "robots": "no-rule",
    "http": "403 (challenge)"
   },
   "ChatGPT-User": {
    "robots": "no-rule",
    "http": "403 (challenge)"
   },
   "ClaudeBot": {
    "robots": "no-rule",
    "http": "403 (challenge)"
   },
   "Claude-SearchBot": {
    "robots": "no-rule",
    "http": "403 (challenge)"
   },
   "Google-Extended": {
    "robots": "no-rule",
    "http": "403 (challenge)"
   },
   "PerplexityBot": {
    "robots": "no-rule",
    "http": "403 (challenge)"
   },
   "CCBot": {
    "robots": "no-rule",
    "http": "403 (challenge)"
   }
  },
  "anthropic.com": {
   "GPTBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "OAI-SearchBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "ChatGPT-User": {
    "robots": "no-rule",
    "http": "200"
   },
   "ClaudeBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "Claude-SearchBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "Google-Extended": {
    "robots": "no-rule",
    "http": "200"
   },
   "PerplexityBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "CCBot": {
    "robots": "no-rule",
    "http": "200"
   }
  },
  "google.com": {
   "GPTBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "OAI-SearchBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "ChatGPT-User": {
    "robots": "no-rule",
    "http": "200"
   },
   "ClaudeBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "Claude-SearchBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "Google-Extended": {
    "robots": "no-rule",
    "http": "200"
   },
   "PerplexityBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "CCBot": {
    "robots": "no-rule",
    "http": "200"
   }
  },
  "perplexity.ai": {
   "GPTBot": {
    "robots": "no-rule",
    "http": "403 (cf-403)"
   },
   "OAI-SearchBot": {
    "robots": "no-rule",
    "http": "403 (challenge)"
   },
   "ChatGPT-User": {
    "robots": "no-rule",
    "http": "200"
   },
   "ClaudeBot": {
    "robots": "no-rule",
    "http": "403 (cf-403)"
   },
   "Claude-SearchBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "Google-Extended": {
    "robots": "no-rule",
    "http": "200"
   },
   "PerplexityBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "CCBot": {
    "robots": "no-rule",
    "http": "403 (challenge)"
   }
  },
  "mistral.ai": {
   "GPTBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "OAI-SearchBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "ChatGPT-User": {
    "robots": "no-rule",
    "http": "200"
   },
   "ClaudeBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "Claude-SearchBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "Google-Extended": {
    "robots": "no-rule",
    "http": "200"
   },
   "PerplexityBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "CCBot": {
    "robots": "no-rule",
    "http": "200"
   }
  },
  "cohere.com": {
   "GPTBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "OAI-SearchBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "ChatGPT-User": {
    "robots": "no-rule",
    "http": "200"
   },
   "ClaudeBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "Claude-SearchBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "Google-Extended": {
    "robots": "no-rule",
    "http": "200"
   },
   "PerplexityBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "CCBot": {
    "robots": "no-rule",
    "http": "200"
   }
  },
  "meta.ai": {
   "GPTBot": {
    "robots": "no-rule",
    "http": "403"
   },
   "OAI-SearchBot": {
    "robots": "no-rule",
    "http": "403"
   },
   "ChatGPT-User": {
    "robots": "no-rule",
    "http": "403"
   },
   "ClaudeBot": {
    "robots": "no-rule",
    "http": "403"
   },
   "Claude-SearchBot": {
    "robots": "no-rule",
    "http": "403"
   },
   "Google-Extended": {
    "robots": "no-rule",
    "http": "403"
   },
   "PerplexityBot": {
    "robots": "no-rule",
    "http": "403"
   },
   "CCBot": {
    "robots": "no-rule",
    "http": "403"
   }
  },
  "x.ai": {
   "GPTBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "OAI-SearchBot": {
    "robots": "no-rule",
    "http": "200"
   },
   "ChatGPT-User": {
    "robots": "no-rule",
    "http": "403 (cf-403)"
   },
   "ClaudeBot": {
    "robots": "no-rule",
    "http": "403 (cf-403)"
   },
   "Claude-SearchBot": {
    "robots": "no-rule",
    "http": "403 (cf-403)"
   },
   "Google-Extended": {
    "robots": "no-rule",
    "http": "403 (cf-403)"
   },
   "PerplexityBot": {
    "robots": "no-rule",
    "http": "403 (cf-403)"
   },
   "CCBot": {
    "robots": "no-rule",
    "http": "200"
   }
  }
 }
}