# BragThat — robots policy
# Goal: keep the landing page indexable for legitimate search engines while
# stopping crawlers from burning serverless compute on dynamic routes.
#
# Every /title/, /person/, /profile/ page is a dynamic SSR render that hits
# Supabase + TMDB. Letting bots fan out across thousands of those routes is
# the fastest way to exhaust Netlify function minutes.

# ── Block aggressive AI / SEO scrapers entirely ─────────────────────────────
# These bots ignore most niceties and crawl very aggressively. The MVP gets
# nothing from being in their indexes.

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

# ── Link preview / unfurler bots ────────────────────────────────────────────
# These are NOT crawlers. Each one fetches exactly the URL a human chose to
# share, once, to build the preview card. They do not fan out across the id
# space, so the compute argument behind the blanket /title/ and /person/
# blocks below does not apply to them.
#
# They must be allowed, and specifically on /title/. Sharing is BragThat's
# primary growth loop; with these blocked, every shared link renders as a bare
# URL with no title, no description and no poster. That was the case until
# 2026-08-12, when Facebook's debugger returned "could be due to a robots.txt
# block" and it turned out to be exactly that.
#
# robots.txt group matching: a bot obeys only the MOST SPECIFIC User-agent
# group that matches it, so these groups exempt each bot from the "*" rules
# further down entirely.

User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: Applebot
Allow: /

# ── Default rules for everyone else (Googlebot, Bingbot, etc.) ──────────────
# Landing page stays crawlable. Dynamic SSR routes are off-limits.

User-agent: *
Disallow: /api/
Disallow: /admin
Disallow: /auth/
Disallow: /discover
Disallow: /feed
# /invite (no slash) covers the admission-gate page AND /invite/{userId}
# referral pages; /i/ covers invite-code deep links.
Disallow: /invite
Disallow: /i/
Disallow: /lists
Disallow: /onboarding
Disallow: /people
Disallow: /person/
# /profile/ crawling is allowed so Founding Creators and other verified
# users are discoverable. Per-page indexation is gated in the page's
# generateMetadata: unverified profiles emit robots.index=false, so
# random low-quality user pages don't clutter search results even though
# Googlebot can still crawl them for canonicalization.
Disallow: /queue
Disallow: /speed-rate
Disallow: /techstars/
Disallow: /title/

# Sitemap — emits the homepage + Privacy + Terms + one /@handle entry per
# verified onboarded profile. See app/sitemap.ts for the exact source.
Sitemap: https://bragthat.com/sitemap.xml