# Allow Google AdSense
User-agent: Mediapartners-Google
Disallow:

# Allow GPTBot (OpenAI's crawler)
User-agent: GPTBot
Allow: /

# Allow ClaudeBot (Anthropic's crawler)
User-agent: ClaudeBot
Allow: /

# Allow PerplexityBot (Perplexity.ai's crawler)
User-agent: PerplexityBot
Allow: /

# Allow CCBot (Common Crawl)
User-agent: CCBot
Allow: /

# Allow Google-Extended (Google's AI crawler)
User-agent: Google-Extended
Allow: /

# Allow Meta's AI crawler
User-agent: meta-externalagent
Allow: /

# The following rules apply to all search engines: block search and archive pages, but allow all blog posts and pages.

# General rules for all other bots
User-agent: *
Disallow: /search*
Disallow: /20*
Allow: /*.html

# Sitemaps
Sitemap: https://www.starklytech.com/sitemap.xml
Sitemap: https://www.starklytech.com/sitemap-pages.xml