# SF Times — robots.txt
# Public site indexing, with internal-only routes blocked.
# AI training crawler disallows match the /standards page commitment.

User-agent: *
Allow: /
Disallow: /admin
Disallow: /focal-audit
Disallow: /tell-your-story
Disallow: /sponsorship-deck
Disallow: /brief/preview
Disallow: /brief-dashboard
Disallow: /brief-dashboard/weekly
Disallow: /brief-dashboard/health

# AI training crawlers. Per /standards, we do not consent to having our
# content used to train commercial AI models. Crawlers that ignore these
# directives are operating without our consent.

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# Search engines that respect crawl-delay get a courtesy one.
User-agent: Bingbot
Crawl-delay: 5

User-agent: Slurp
Crawl-delay: 10

Sitemap: https://www.sftimes.com/sitemap.xml
Sitemap: https://www.sftimes.com/news-sitemap.xml