User-agent: *
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /assets/
# Block .md files from search engines (LLM-only content, prevent duplicate indexing)
Disallow: /*.md$
Disallow: /llms-full.txt

# Search engine crawlers - full access (including /assets/ for rendering)
User-agent: Googlebot
Allow: /assets/
Disallow: /*.md$
Disallow: /llms-full.txt
Allow: /

User-agent: Bingbot
Allow: /assets/
Disallow: /*.md$
Disallow: /llms-full.txt
Allow: /

# AI answer engines - allow EVERYTHING including .md and llms files
User-agent: GPTBot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /*.md$
Allow: /

User-agent: ClaudeBot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /*.md$
Allow: /

User-agent: PerplexityBot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /*.md$
Allow: /

# AI training crawlers - BLOCK (protect original content from model training)
User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: cohere-ai
Disallow: /

# Block SEO spy/scraping tools
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

Sitemap: https://www.graygroupintl.com/sitemap.xml
Sitemap: https://www.graygroupintl.com/news-sitemap.xml
Sitemap: https://www.graygroupintl.com/feed.xml

# LLM content index
# See https://llmstxt.org/ for the llms.txt specification
Llmstxt: https://www.graygroupintl.com/llms.txt