# robots.txt — gbcorp.org
# Policy: open for search + live AI fetchers; closed to AI training crawlers.
# Cloudflare AI Content Signals header (interpreted by Cloudflare-aware bots).

# ─── Default policy: anything not listed below ─────────────────────────
# Static site on Firebase Hosting (WordPress retired 2026-07) — no WP paths.
# /search/ = internal search results; blocked per Google guidance (crawl budget).
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /search/

# ─── Regular search engines (allowed) ──────────────────────────────────
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# ─── Live AI fetchers (allowed — they cite us with link-backs) ─────────
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: DuckAssistBot
Allow: /

# ─── AI training crawlers (blocked — keeps our IP out of model weights) ─
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: PetalBot
Disallow: /

# ─── Sitemaps ──────────────────────────────────────────────────────────
# Sitemap index points to static-sitemap.xml + articles-sitemap.xml +
# blog-sitemap.xml. Static = all non-article pages (stable); articles/blog
# refresh when new posts ship. Submit sitemap_index.xml to GSC; it discovers all.
Sitemap: https://gbcorp.org/sitemap_index.xml
Sitemap: https://gbcorp.org/static-sitemap.xml
Sitemap: https://gbcorp.org/articles-sitemap.xml
Sitemap: https://gbcorp.org/blog-sitemap.xml