# Dubai Generator Rental Online — robots.txt
# 20–1250 kVA Diesel Generator Sizing, DEWA 400V Amperage & 50°C Thermal Derating Portal — dubaigeneratorrental.online
# Last updated: October 2026

# Allow all standard search engines
User-agent: *
Allow: /

# Sitemaps
Sitemap: https://dubaigeneratorrental.online/sitemap-index.xml

# AI/LLM Crawlers — Selectively Permitted
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

# Blocked AI Crawlers (no attribution, aggressive scraping)
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

# References
# LLMs-Full: https://dubaigeneratorrental.online/llms-full.txt
# LLMs: https://dubaigeneratorrental.online/llms.txt
# Media-RSS: https://dubaigeneratorrental.online/rss.xml