###############################
#
#
User-agent: *
#
# list folders robots are not allowed to index 
#

User-agent: msnbot
Crawl-delay: 10

User-agent: Teoma
Crawl-delay: 10

User-agent: Slurp
Crawl-delay: 10

User-agent: IconSurf
Disallow: /

User-agent: aipbot
Disallow: /

User-agent: BecomeBot
Disallow: /

User-agent: psbot
Disallow: /

Disallow: /admin/

Disallow: /cgi-bin/

Disallow: /cp/

Disallow: /ctrl/

Disallow: /obj/

Disallow: /OldSite/

Disallow: /site_dev/

Disallow: /MapSearch/

Disallow: /download/

#
# list specific files robots are not allowed to index
#
#Disallow: /directory/foo.html
#Disallow: /folder2/whatever.html
# 
#
###############################