Enter a domain. We read its robots.txt and show, crawler by crawler, whether AI bots can crawl you — plus a config you can paste.
# 禁止下列 AI 爬虫抓取本站 User-agent: GPTBot Disallow: / User-agent: OAI-SearchBot Disallow: / User-agent: ChatGPT-User Disallow: / User-agent: ClaudeBot Disallow: / User-agent: Claude-User Disallow: / User-agent: Claude-SearchBot Disallow: / User-agent: Google-Extended Disallow: / User-agent: CCBot Disallow: / User-agent: PerplexityBot Disallow: / User-agent: Bytespider Disallow: / User-agent: Amazonbot Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: meta-externalagent Disallow: / User-agent: cohere-ai Disallow: / User-agent: YouBot Disallow: /
# 禁止「训练」类爬虫，保留「搜索」类（不影响被搜索引擎收录） User-agent: GPTBot Disallow: / User-agent: ClaudeBot Disallow: / User-agent: Google-Extended Disallow: / User-agent: CCBot Disallow: / User-agent: Bytespider Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: meta-externalagent Disallow: / User-agent: cohere-ai Disallow: /
# 放在 http {} 块里
map $http_user_agent $block_ai_bot {
default 0;
"~*(GPTBot|OAI-SearchBot|ChatGPT-User|ClaudeBot|Claude-User|Claude-SearchBot|Google-Extended|CCBot|PerplexityBot|Bytespider|Amazonbot|Applebot-Extended|meta-externalagent|cohere-ai|YouBot)" 1;
}
# 放在 server {} 块里
if ($block_ai_bot) {
return 403;
}