User-agent: Sproose Disallow: / User-agent: Fasterfox Disallow: / User-agent: HTTrack Disallow: / User-agent: MSR-ISRCCrawler Disallow: / User-agent: SiteSucker Disallow: / User-agent: WebReaper Disallow: / # Copied these from http://en.wikipedia.org/robots.txt User-agent: UbiCrawler Disallow: / User-agent: DOC Disallow: / User-agent: Zao Disallow: / User-agent: sitecheck.internetseer.com Disallow: / User-agent: Zealbot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: linko Disallow: / User-agent: Microsoft.URL.Control Disallow: / User-agent: Xenu Disallow: / User-agent: larbin Disallow: / User-agent: ZyBORG Disallow: / User-agent: Download Ninja Disallow: / # The 'grub' distributed client has been *very* poorly behaved. User-agent: grub-client Disallow: / # Doesn't follow robots.txt anyway, but... User-agent: k2spider Disallow: / # http://www.nameprotect.com/botinfo.html User-agent: NPBot Disallow: / User-agent: * Disallow: /rvebay.htm Disallow: /francis.htm Disallow: /shop/faqcontribute.php Disallow: /boards/login.php Disallow: /boards/memberlist.php Disallow: /boards/posting.php Disallow: /boards/privmsg.php Disallow: /boards/profile.php Disallow: /boards/search.php Disallow: /boards/ads_comment_delete.php Disallow: /boards/ads_comment_edit.php Disallow: /boards/ads_comment_temp.php Disallow: /boards/ads_create.php Disallow: /boards/ads_images.php Disallow: /boards/ads_item_delete.php Disallow: /boards/ads_item_edit.php Disallow: /boards/ads_item_renewal.php Disallow: /boards/ads_item_status.php Disallow: /boards/ads_popup.php Disallow: /boards/ads_search.php Disallow: /Newlook1.htm Disallow: /NewlookDone.htm Disallow: /cgi-bin/ Crawl-delay: 10