# socii.dk
#
# The vendor rules below the first block are SNgine's. Everything above them
# is socii's, and exists because crawling was measured at 627,000 post views
# a day across 10,000 posts from 64,000 addresses — against roughly 8,000
# accounts, a dozen of which had actually read the post that prompted the
# look.
#
# Search engines are welcome; members' posts being findable is the point.
# What is declined below is bulk collection for training corpora and
# commercial SEO indexes, neither of which is why anyone posted here.
#
# This file is a request, and the crawlers generating the most load are the
# least likely to honour it. It is the polite half of the answer.

# ── Training-corpus collection ───────────────────────────────────────────────
# Google-Extended and Applebot-Extended are AI opt-outs only: declining them
# does not affect Google Search or Siri results.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Bytespider
User-agent: Amazonbot
User-agent: CCBot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Omgili
User-agent: Timpibot
User-agent: Webzio-Extended
Disallow: /

# ── Commercial SEO and backlink indexes ──────────────────────────────────────
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DataForSeoBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: PetalBot
User-agent: Barkrowler
User-agent: serpstatbot
Disallow: /

# ── Everyone else ────────────────────────────────────────────────────────────
# Crawl-delay is not in the standard and Google ignores it; Bing and Yandex
# honour it, and it costs nothing to ask.
User-agent: *
Crawl-delay: 10
Disallow: /content/backups
Disallow: /content/languages
Disallow: /includes/libs
Disallow: /install
Disallow: /admincp
Disallow: /modcp