robots.txt

# Common Crawl robot, the resulting dataset is the primary training corpus in every LLM.
User-agent: CCBot
Disallow: /

# ChatGPT robot, used to improve the ChatGPT LLM. 
User-agent: ChatGPT-User
Disallow: /

# ChatGPT robot, may be used to improve the ChatGPT LLM. 
User-agent: GPTBot
Disallow: /

# Robot used to improve Bard and Vertex AI LLMs.
User-agent: Google-Extended
Disallow: /

# webz.io robot, the resulting dataset can and is purchased to train LLMs.
User-agent: Omgilibot
Disallow: /

# webz.io robot, the resulting dataset can and is purchased to train LLMs.
User-agent: Omgili
Disallow: /

# FacebookBot crawls public web pages to improve LLMs for Facebook's speech recognition technology. 
User-agent: FacebookBot
Disallow: /

# Amazonbot is used to train Amazon services such as Alexa.
User-agent: Amazonbot
Disallow: /

# Bytespider is ByteDance's bot and may not respect robots.txt.
User-agent: Bytespider
Disallow: /

# Robot used to improve Anthropic AI LLMs.
User-agent: anthropic-ai
Disallow: /

# OpenAI search bot 
User-agent: OAI-SearchBot
Disallow: /

# Velen.io/Hunter.io "build business datasets and machine learning models to better understand the web" - seems to focus on collecting email adresses for spam though. 
User-agent: VelenPublicWebCrawler
Disallow: /