User-agent: *
Allow: /

# Sitemap locations
Sitemap: https://kaaldev.com/sitemap.xml
Sitemap: https://kaaldev.com/ai-sitemap.xml
Sitemap: https://kaaldev.com/fivem-sitemap.xml
Sitemap: https://kaaldev.com/keywords-sitemap.xml

# AI Discovery file
# AI-specific robots.txt: https://kaaldev.com/ai-robots.txt
# AI discovery metadata: https://kaaldev.com/ai-discovery.json

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Allow all major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# Block access to sensitive files
Disallow: /private/
Disallow: /*.log$
Disallow: /temp/
Disallow: /cache/

# Allow access to CSS and JS files for proper rendering
Allow: /styles.css
Allow: /script.js
Allow: /public/images/

# Specifically allow SEO and keyword files for better indexing
Allow: /kaaldev-keywords.json
Allow: /fivem-seo-implementation-strategy.md
Allow: /ai-discovery.json

# FiveM and gaming related content
User-agent: Googlebot
Allow: /kaaldev-keywords.json
Allow: /fivem-seo-implementation-strategy.md
Crawl-delay: 1

User-agent: Bingbot
Allow: /kaaldev-keywords.json
Allow: /fivem-seo-implementation-strategy.md
Crawl-delay: 1