# Robots.txt for Traffic Exchanges Platform
# Generated on 15 October 2025

# Allow all search engines to crawl all content
User-agent: *
Allow: /

# Allow specific language pages
Allow: /tr
Allow: /es  
Allow: /fr
Allow: /ru

# Allow CSS, JS, and image files
Allow: /assets/
Allow: /*.css
Allow: /*.js
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.svg
Allow: /*.ico

# Disallow admin/private areas (if any exist in future)
Disallow: /admin/
Disallow: /private/
Disallow: /config/
Disallow: /.env
Disallow: /vendor/

# Allow common SEO files
Allow: /sitemap.xml
Allow: /sitemap*.xml
Allow: /favicon.png

# Crawl delay (optional - helps with server load)
Crawl-delay: 1

# Sitemap location
Sitemap: https://trafficexchanges.vercel.app/sitemap.xml

# Special rules for different search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# Block spam bots and scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /