# Generic robots.txt file, used unless a site needs to specify their own

# We block access to the stats pages (which should be password-protected anyway)
User-agent: *
Disallow: /stats/

# MSNbot causes problems because it indexes too fast, so slow it *right* down
User-agent: msnbot
Crawl-delay:  20
# We need to re-specifiy the disallow, too
Disallow: /stats/