# togetajob.com
#
# Owner's policy (2026-10-03): search engines and AI assistants may read this
# site to answer questions and cite it, with a link. Collecting it to train
# AI models is not allowed.
#
# Content Signals (contentsignals.org): search = search results and links,
# ai-input = an AI answer that reads a page live and cites it,
# ai-train = training or fine-tuning a model.
Content-Signal: search=yes, ai-input=yes, ai-train=no
User-agent: *
Allow: /

# Crawlers that gather pages for model training, by their published names.
# Their companies' search and on-demand agents (OAI-SearchBot, ChatGPT-User,
# Claude-SearchBot, Claude-User, PerplexityBot, Perplexity-User, Googlebot,
# Bingbot, Applebot) fall under the rule above and are allowed.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Meta-ExternalAgent
User-agent: Bytespider
User-agent: Amazonbot
User-agent: cohere-training-data-crawler
User-agent: Diffbot
Disallow: /

Sitemap: https://togetajob.com/sitemap.xml