# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
User-Agent: *
Disallow: /

User-agent: discobot
Disallow: / 

User-agent: MLBot
Disallow: / 

# Keep search engines out directories that have no search value.
User-Agent: *
Disallow: /block/
Disallow: /flickr/
Disallow: /delicious/
Disallow: /live_update/
Disallow: /api/