# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#

# Disable (Expedia?) security scanning since it hammers our site
User-agent: Acunetix Security Scanner
Disallow: /

User-Agent: *
Disallow: /widgets/lists/*/info
Disallow: /lists/*/discoveries
Disallow: /discoveries/wall_map
Disallow: /login_callback
Crawl-delay: 0.5 

User-agent: Googlebot
Disallow: /widgets/lists/*/info
Disallow: /lists/*/discoveries
Disallow: /discoveries/wall_map
Disallow: /login_callback
Crawl-delay: 0

User-agent: AhrefsBot
Disallow: /widgets/lists/*/info
Disallow: /lists/*/discoveries
Disallow: /discoveries/wall_map
Disallow: /login_callback
Crawl-delay: 4

User-agent: QSS_1
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: MJ12bot
Disallow: /


Sitemap: https://www.trover.com/sitemaps/default_sitemap.xml.gz
Sitemap: https://www.trover.com/sitemaps/lists_sitemap.xml.gz
Sitemap: https://www.trover.com/sitemaps/discoveries_sitemap.xml.gz
Sitemap: https://www.trover.com/sitemaps/cities_sitemap.xml.gz
Sitemap: https://www.trover.com/sitemaps/establishments_sitemap.xml.gz
Sitemap: https://www.trover.com/sitemaps/interests_sitemap.xml.gz
Sitemap: https://www.trover.com/sitemaps/users_sitemap.xml.gz