# ============================================
# XForex Robots.txt
# Hosted permanently on Cloudflare Pages
# Last updated: 2026-08-01
# ============================================

# Default rules for all crawlers
User-agent: *
Allow: /
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.ico$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.ttf$

# Block access to sensitive or non-public paths (if any)
Disallow: /admin/
Disallow: /api/
Disallow: /internal/
Disallow: /tmp/
Disallow: /private/
Disallow: /*?*password=
Disallow: /*?*token=
Disallow: /*?*key=

# Crawl-delay (optional, polite behavior)
Crawl-delay: 1

# ============================================
# Google-specific directives
# ============================================
User-agent: Googlebot
Allow: /
Allow: /*.js$
Allow: /*.css$
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Mediapartners-Google
Allow: /

# ============================================
# Yahoo-specific directives
# ============================================
User-agent: Slurp
Allow: /
Crawl-delay: 1

User-agent: Yahoo! Slurp
Allow: /
Crawl-delay: 1

User-agent: YahooSeeker
Allow: /

User-agent: Yahoo-MMCrawler
Allow: /

# ============================================
# Bing / MSN (good practice for broader indexing)
# ============================================
User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: msnbot
Allow: /

# ============================================
# AI Crawlers (optional - allow or block)
# ============================================
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

# ============================================
# Block known bad bots & scrapers
# ============================================
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /

# ============================================
# Sitemap location
# ============================================
Sitemap: https://xforex.pages.dev/sitemap.xml

# Additional sitemap for images (if created later)
# Sitemap: https://xforex.pages.dev/sitemap-images.xml