#Robots.txt

#disallow directory 
user-agent: *
disallow: /cgi-bin/

user-agent: *
disallow: /images/

user-agent: *
disallow: /includes/

user-agent: *
disallow: /log/

#disallow indexing of specific file extensions
user-agent: *
disallow: /*.gif$

user-agent: *
disallow: /*.jpg$

user-agent: *
disallow: /*.zip$

user-agent: *
disallow: /*.doc$

user-agent: *
disallow: *.exe$

user-agent: *
duisallow: /*.pdf$

#Disallow Collectors and Spam
user-agent: atSpider
disallow: /

user-agent: charrypicker
disallow: /

user-agent: DSirf
disallow: /

user-agent: EliteSys Entry
disallow: /

user-agent: EmailCollector
disallow: /

user-agent: EmailSiphon
disallow: /

user-agent: EmailWolf
disallow: /

user-agent: Mail Sweeper
disallow: /

user-agent: munky
disallow: /

user-agent: Roverbot
disallow: /

user-agent: WebEmailExtrac
disallow: /

#Disallow Offline Browsers
user-agent: xGet
disallow: /

user-agent: wGet
disallow: /

user-agent: WebWalk
disallow: /

user-agent: webvac
disallow: /

user-agent: WebReaper
disallow: /

user-agent: WebMirror
disallow: /

user-agent: WebFetcher
disallow: /

user-agent: WebCopy
disallow: /

user-agent: webcopier
disallow: /

user-agent: WebCatcher
disallow: /

user-agent: WebBandit
disallow: /

user-agent: w3mir
disallow: /

user-agent: vobsub
disallow: /

user-agent: Templeton
disallow: /

user-agent: ssearcher100
disallow: /

user-agent: SpiderBot
disallow: /

user-agent: Shai'Hulud
disallow: /

user-agent: PBWF
disallow: /

user-agent: LightningDownload
disallow: /

user-agent: KDD Exploror
disallow: /

user-agent: Jeeves
disallow: /

user-agent: Internet Explore
disallow: /

user-agent: httrack
disallow: /

user-agent: HavIndex
disallow: /

User-agent: GetUrl
disallow: /

user-agent: GetBot
disallow: /

user-agent: ESIRover
disallow: /

user-agent: Download Wonder
disallow: /

user-agent: Collage
disallow: /

user-agent: Mozilla/2.0 (compatible; MS FrontPage 4.0)
disallow: /