# https://qa.tech/robots.txt
# Disallow path prefixes: keep in sync with prerender/seo-config.mjs
# NOTE: /try-now/*, /techarena, and /qa.tech/codehead are intentionally NOT
# disallowed here. Those routes serve `X-Robots-Tag: noindex, nofollow` (see
# vercel.json) plus a `<meta robots>` tag in-page. Crawlers must be allowed
# to fetch them in order to see the noindex directive; blocking via
# robots.txt would leave URL-only entries in the search index.

# --- Traditional search engines ---------------------------------------------
User-agent: Googlebot
Allow: /
Disallow: /design-system
Disallow: /_variants
Disallow: /_drafts
Disallow: /_wip

User-agent: Bingbot
Allow: /
Disallow: /design-system
Disallow: /_variants
Disallow: /_drafts
Disallow: /_wip

# --- Social previews --------------------------------------------------------
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# --- AI / LLM crawlers (explicit) -------------------------------------------
# OpenAI: training crawler, search index crawler, on-demand user fetches
User-agent: GPTBot
Allow: /
Disallow: /design-system
Disallow: /_variants
Disallow: /_drafts
Disallow: /_wip

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic: training crawler, search index crawler, on-demand user fetches
User-agent: ClaudeBot
Allow: /
Disallow: /design-system
Disallow: /_variants
Disallow: /_drafts
Disallow: /_wip

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Perplexity: index crawler + on-demand user fetches
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google Gemini / AI Overviews opt-in (separate from Googlebot)
User-agent: Google-Extended
Allow: /

# Apple Intelligence / Apple AI opt-in (separate from Applebot)
User-agent: Applebot-Extended
Allow: /

# --- Default ----------------------------------------------------------------
# Content Signals (https://contentsignals.org): how this site's content may be
# used. Search indexing, AI-assisted answers with attribution, and AI training
# are all permitted.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /design-system
Disallow: /_variants
Disallow: /_drafts
Disallow: /_wip

Sitemap: https://qa.tech/sitemap.xml

# LLM-readable site map (markdown)
# llms.txt: https://qa.tech/llms.txt
#
# Every indexable page also has a markdown twin: append `.md` to the path
# (https://qa.tech/blog/<slug>.md), or send `Accept: text/markdown`.
#
# Agent discovery:
# MCP server card: https://qa.tech/.well-known/mcp/server-card.json
# API catalog (RFC 9727): https://qa.tech/.well-known/api-catalog
# Agent skills: https://qa.tech/.well-known/agent-skills.json
# API authentication: https://qa.tech/.well-known/auth.md
# Developer hub: https://qa.tech/developers