User-agent: *
# BLOCK standard paths
Disallow: /ghost/
Disallow: /email/
Disallow: /tag/
Disallow: /sitemap.xml
Disallow: /sitemap-tags.xml
Disallow: /admin
Disallow: /database

# Explicitly allow primary search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# BLOCK AI Training Bots (Content scraping for model building)
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

# ALLOW AI Reference/Citation Bots (These drive referral traffic)
User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

# Sitemaps that can be indexed
Sitemap: https://www.security.land/sitemap-posts.xml
Sitemap: https://www.security.land/news-sitemap.xml
Sitemap: https://www.security.land/sitemap-pages.xml