# robots.txt for http://www.barkersvale.com.au/ # comments to webmaster@bigvolcano.com.au # 04/11/08 # crawler delay list starts here User-agent: Googlebot Crawl-delay: 10 User-agent: Slurp Crawl-delay: 10 User-agent: msnbot Crawl-delay: 10 User-agent: Teoma Crawl-delay: 10 User-agent: * Crawl-delay: 20 # crawler delay list ends here # image bot general list starts here User-agent: Googlebot-Image/1.0 Disallow: / User-agent: Googlebot-Image/ Disallow: / User-agent: psbot Disallow: / User-agent: sdcresearchlabs-testbot Disallow: / # image bot list ends here # uptime bots start here User-agent: sitecheck.internetseer.com Disallow: / User-agent: Uptimebot Disallow: / User-agent: linkwalker Disallow: / # uptime bots end here # general list starts here User-agent: www.findsounds.com Disallow: / User-agent: Nutch Disallow: / User-agent: SBIder Disallow: / User-agent: grub-client Disallow: / User-agent: SiteSucker Disallow: / User-agent: SurveyBot Disallow: / User-agent: lachesis Disallow: / User-agent: NPBot Disallow: / User-agent: NPBot-1/2.0 Disallow: / User-agent: linksmanager Disallow: / # general list ends # directed to all robots User-agent: * Disallow: /ftp/ Disallow: /cgi-bin/ Disallow: /errormsg/ Disallow: /js/ Disallow: /ssl/ Disallow: /css/ Disallow: /images/ Disallow: /media/2008march/ Disallow: /media/BV photos/ Disallow: /media/photoalbum/images/ Disallow: /media/photoalbum/thumbnails/ Disallow: /media/photos-oldsite/ Disallow: /sounds/ Disallow: thanks.htm