# Kubeez robots.txt
# https://kubeez.com

# ============================================================
# DEFAULT RULES — All crawlers
# ============================================================
User-agent: *
Disallow: /admin/
Disallow: /dev/
Disallow: /staging/
Disallow: /api/
Disallow: /auth/
Disallow: /static/
Disallow: /@react-refresh
Disallow: /admin-dashboard
Disallow: /dev-admin
# Admin template library (requires auth; avoid crawl budget on app shell)
Disallow: /media/library/admin

# Block query strings globally (prevents duplicate content)
Disallow: /*?*

# Allow public pages explicitly (overrides query string block above)
Allow: /$
Allow: /dashboard
Allow: /images$
Allow: /video-generation$
Allow: /audio
Allow: /ads$
Allow: /pricing$
Allow: /docs
Allow: /blog
Allow: /legal/
Allow: /media
Allow: /share/
Allow: /features$
Allow: /use-cases
Allow: /changelog$
Allow: /sitemap$

# Allow LLM/AI crawler discovery files
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai.txt
Allow: /sitemap-images.xml

# Allow favicon and PWA assets (required for Google Search favicon display)
Allow: /favicon.ico
Allow: /favicon.png
Allow: /favicon.svg
Allow: /favicon-96x96.png
Allow: /android-chrome-192x192.png
Allow: /android-chrome-512x512.png
Allow: /apple-touch-icon.png
Allow: /site.webmanifest
Allow: /og-image.jpg

# ============================================================
# AI CRAWLERS — Allow full access to public content
# These bots power ChatGPT, Perplexity, Gemini, Claude, etc.
# ============================================================

User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: Claude-Web
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: Googlebot-Extended
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: cohere-ai
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

User-agent: YouBot
Allow: /
Disallow: /admin/
Disallow: /media/library/admin
Disallow: /auth/
Disallow: /api/

# ============================================================
# AGGRESSIVE BOTS — Crawl-delay to protect server
# ============================================================

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

# ============================================================
# PREFERRED DOMAIN (Bing/Yahoo)
# ============================================================
Host: kubeez.com

# ============================================================
# SITEMAPS
# ============================================================
Sitemap: https://kubeez.com/sitemap.xml
Sitemap: https://kubeez.com/sitemap-images.xml