# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file # Allow all crawlers User-agent: * Allow: / # Disallow authenticated areas Disallow: /dashboard Disallow: /admin Disallow: /users/ Disallow: /badge_requests Disallow: /listings Disallow: /analytics Disallow: /settings # Sitemap location Sitemap: <%= ENV['SITE_URL'] || 'http://localhost:3000' %>/sitemap.xml