User-agent: *
Allow: /

# Important pages that actually exist
Allow: /search
Allow: /explore
Allow: /features  
Allow: /pricing
Allow: /fasttrack
Allow: /rail-storage
Allow: /transload
Allow: /rail-services
Allow: /real-estate
Allow: /ports
Allow: /intermodal
Allow: /trucking

# Sitemaps
Sitemap: https://standardrail.com/sitemap.xml
Sitemap: https://standardrail.com/api/sitemap

# Crawl delay
Crawl-delay: 1

# Block admin areas
Disallow: /admin/
Disallow: /dashboard/
Disallow: /api/
Disallow: /login
Disallow: /signup
Disallow: /investor

# Block private user areas
Disallow: /profile/
Disallow: /settings/
Disallow: /billing/

# Block development/test paths
Disallow: /test/
Disallow: /dev/
Disallow: /_next/
Disallow: /static/

# Allow social media crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Block specific bots if needed
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10