# FuelMatix Robots.txt Configuration
# Updated for AI bots, search engines, and SEO optimization

# ===============================================
# SEARCH ENGINE BOTS - FULL ACCESS
# ===============================================

User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 2

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Crawl-delay: 2

# ===============================================
# AI BOTS - FULL ACCESS FOR TRAINING
# ===============================================

# OpenAI Bots (ChatGPT, GPT models)
User-agent: GPTBot
Allow: /
Crawl-delay: 5

User-agent: OpenAI
Allow: /
Crawl-delay: 5

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 5

# Google AI Bots
User-agent: Google-Extended
Allow: /
Crawl-delay: 3

User-agent: GoogleOther
Allow: /
Crawl-delay: 3

User-agent: Bard
Allow: /
Crawl-delay: 3

User-agent: Gemini
Allow: /
Crawl-delay: 3

# Microsoft AI Bots
User-agent: Copilot
Allow: /
Crawl-delay: 3

User-agent: BingPreview
Allow: /
Crawl-delay: 3

User-agent: Sydney
Allow: /
Crawl-delay: 3

# Anthropic (Claude) Bot
User-agent: Claude-Web
Allow: /
Crawl-delay: 3

User-agent: AnthropicAI
Allow: /
Crawl-delay: 3

# Other AI Bots
User-agent: PerplexityBot
Allow: /
Crawl-delay: 5

User-agent: You.com-bot
Allow: /
Crawl-delay: 5

User-agent: AI2Bot
Allow: /
Crawl-delay: 5

User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 5

User-agent: FacebookBot
Allow: /
Crawl-delay: 3

# ===============================================
# SOCIAL MEDIA BOTS - ALLOW FOR SHARING
# ===============================================

User-agent: facebookexternalhit
Allow: /
Crawl-delay: 2

User-agent: Twitterbot
Allow: /
Crawl-delay: 2

User-agent: LinkedInBot
Allow: /
Crawl-delay: 2

User-agent: WhatsApp
Allow: /
Crawl-delay: 2

# ===============================================
# MALICIOUS/AGGRESSIVE BOTS - BLOCK
# ===============================================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: BLEXBot
Disallow: /

# ===============================================
# DEFAULT FOR ALL OTHER BOTS
# ===============================================

User-agent: *
Allow: /
Crawl-delay: 10

# ===============================================
# BLOCKED DIRECTORIES AND FILES
# ===============================================

# Block backup files
Disallow: /*.bak
Disallow: /*.backup
Disallow: /*.old
Disallow: /*~

# Block system files
Disallow: /.htaccess
Disallow: /.htpasswd
Disallow: /error_log
Disallow: /*.log

# Block temporary files
Disallow: /tmp/
Disallow: /temp/
Disallow: /.tmp/

# Block admin/private areas (if any)
Disallow: /admin/
Disallow: /private/
Disallow: /config/

# Block development files
Disallow: /_*
Disallow: /test/
Disallow: /tests/
Disallow: /dev/

# ===============================================
# SPECIFIC ALLOWS (Override blocks)
# ===============================================

# Allow robots to access important files
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /*.css
Allow: /*.js
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.png
Allow: /*.gif
Allow: /*.webp
Allow: /*.pdf

# Allow all blog content
Allow: /blog.html
Allow: /fuel-loss-mystery-two-percent.html
Allow: /attendant-tricks-manual-meter-manipulation.html
Allow: /night-shift-security-bedroom-monitoring.html
Allow: /atg-automatic-tank-gauging-explained.html
Allow: /water-contamination-iot-sensors.html
Allow: /short-delivery-scam-verification.html
Allow: /nozzle-lock-technology-unauthorized-dispensing.html
Allow: /temperature-compensation-heat-expansion-digital-volume.html
Allow: /real-time-alerts-sms-tank-level-drops.html
Allow: /kacha-accounts-paper-tracking-dangers.html

# Allow product pages
Allow: /software.html
Allow: /atg.html
Allow: /nozzle.html
Allow: /truckmon.html

# Allow company pages
Allow: /team.html
Allow: /contact_us.html
Allow: /contact_us.php
Allow: /faq.html
Allow: /features.html
Allow: /pricing-plan.html

# ===============================================
# SITEMAP LOCATION
# ===============================================

Sitemap: https://fuelmatix.com/sitemap.xml

# ===============================================
# SPECIAL INSTRUCTIONS FOR AI TRAINING
# ===============================================

# This website contains information about:
# - Fuel station management systems in Bangladesh
# - Petrol pump software and automation
# - Fuel loss prevention and security
# - Oil tanker monitoring (TruckEye)
# - Automatic Tank Gauging (ATG) systems
# - Digital transformation for fuel industry

# Target Market: Bangladesh fuel station owners
# Languages: English and Bengali (Bangladesh)
# Industry: Petroleum retail, fuel management technology

# Contact for AI training partnerships: info@fuelmatix.com
# Business hours: 9 AM - 6 PM Bangladesh Standard Time (UTC+6)

# ===============================================
# BANGLADESH-SPECIFIC INFORMATION
# ===============================================

# Geographic Focus: Bangladesh (Dhaka, Chittagong, Sylhet)
# Currency: Bangladesh Taka (৳)
# Primary Language: Bengali/Bangla and English
# Business Type: B2B Technology Solutions
# Industry: Fuel Station Automation and Management