# These settings will keep spiders from indexing your unwanted pages # This assumes that your OSC install is in your web site's ROOT directory # ie: http://www.yoursite.com/index.php <- Use if this brings up your OSC main page Disallow: /includes Disallow: /cgi-bin Disallow: /account.php Disallow: /account_edit.php Disallow: /account_history.php Disallow: /account_history_info.php Disallow: /account_password.php Disallow: /add_checkout_success.php Disallow: /address_book.php Disallow: /address_book_process.php Disallow: /advanced_search.php Disallow: /checkout_confirmation.php Disallow: /checkout_payment.php Disallow: /checkout_payment_address.php Disallow: /checkout_process.php Disallow: /checkout_shipping.php Disallow: /checkout_shipping_address.php Disallow: /checkout_success.php Disallow: /contact_bean.php Disallow: /cookie_usage.php Disallow: /create_account.php Disallow: /create_account_success.php Disallow: /login.php Disallow: /password_forgotten.php Disallow: /popup_image.php Disallow: /shopping_cart.php Disallow: /product_reviews_write.php # Feel free to add any other pages on your site that you don't want to be indexed by # the search engines. # PLEASE NOTE: Any pages that you list here should be secured by other means if you # don't want people to be able to view them, as some malicious users will look at a # robots.txt file to try to find "hidden" or "secret" areas of web sites to find # confidential information. # IF YOU DO NOT WISH TO HAVE THE GOOGLE IMAGE BOT SCAN YOUR DOMAIN FOR IMAGES # THEN YOU CAN INCLUDE THE FOLLOWING IN YOUR ROBOTS FILE. # I FOUND THAT MY BANDWIDTH USAGE DROPPED BY A MASSIVE AMOUNT AFTER I GOT RID # OF THE GOOGLE IMAGE BOT. ALL I HAD WAS IMAGE HUNTERS STEALING PRODUCT SHOTS # AND NOT EVEN BROWSING THE SITE. # User-agent: Googlebot-Image # Disallow: / User-agent: Intelliseek Disallow: / User-agent: BaiDuSpider Disallow: / User-agent: DFusionBot Disallow: / User-agent: Gaisbot Disallow: / User-agent: Girafabot Disallow: / User-agent: polybot Disallow: / User-agent: Szukacz Disallow: / User-agent: webcrawler at wise-guys Disallow: / User-agent: NaverRobot Disallow: / User-agent: vspider Disallow: / User-agent: http://www.almaden.ibm.com/cs/crawler Disallow: / User-agent: Microsoft Scheduled Cache Content Download Service Disallow: / # robots we really don't want User-agent: SlySearch Disallow: / User-agent: turnitinbot Disallow: / User-agent: BlackWidow Disallow: / User-agent: ChinaClaw Disallow: / User-agent: Custo Disallow: / User-agent: DISCo Disallow: / User-agent: Download Demon Disallow: / User-agent: eCatch Disallow: / User-agent: EirGrabber Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: Express WebPictures Disallow: / User-agent: ExtractorPro Disallow: / User-agent: EyeNetIE Disallow: / User-agent: FlashGet Disallow: / User-agent: GetRight Disallow: / User-agent: GetWeb! Disallow: / User-agent: Go!Zilla Disallow: / User-agent: Go-Ahead-Got-It Disallow: / User-agent: GrabNet Disallow: / User-agent: Grafula Disallow: / User-agent: HMView Disallow: / User-agent: HTTrack Disallow: / User-agent: Image Stripper Disallow: / User-agent: Image Sucker Disallow: / User-agent: Indy Library Disallow: / User-agent: InterGET Disallow: / User-agent: Internet Ninja Disallow: / User-agent: JetCar Disallow: / User-agent: JOC Web Spider Disallow: / User-agent: larbin Disallow: / User-agent: LeechFTP Disallow: / User-agent: Mass Downloader Disallow: / User-agent: MIDown tool Disallow: / User-agent: Mister PiX Disallow: / User-agent: Navroad Disallow: / User-agent: NearSite Disallow: / User-agent: NetAnts Disallow: / User-agent: NetSpider Disallow: / User-agent: Net Vampire Disallow: / User-agent: NetZIP Disallow: / User-agent: Octopus Disallow: / User-agent: PageGrabber Disallow: / User-agent: Papa Foto Disallow: / User-agent: pavuk Disallow: / User-agent: pcBrowser Disallow: / User-agent: RealDownload Disallow: / User-agent: ReGet Disallow: / User-agent: SiteSnagger Disallow: / User-agent: SmartDownload Disallow: / User-agent: SuperBot Disallow: / User-agent: SuperHTTP Disallow: / User-agent: Surfbot Disallow: / User-agent: tAkeOut Disallow: / User-agent: Teleport Pro Disallow: / User-agent: VoidEYE Disallow: / User-agent: Web Image Collector Disallow: / User-agent: Web Sucker Disallow: / User-agent: WebAuto Disallow: / User-agent: WebCopier Disallow: / User-agent: WebFetch Disallow: / User-agent: WebGo IS Disallow: / User-agent: WebLeacher Disallow: / User-agent: WebReaper Disallow: / User-agent: WebSauger Disallow: / User-agent: Website eXtractor Disallow: / User-agent: Website Quester Disallow: / User-agent: WebStripper Disallow: / User-agent: WebWhacker Disallow: / User-agent: WebZIP Disallow: / User-agent: Wget Disallow: / User-agent: Widow Disallow: / User-agent: WWWOFFLE Disallow: / User-agent: Xaldon WebSpider Disallow: / User-agent: Zeus Disallow: / User-agent: NPBot Disallow: / User-agent: Test Spider 0.3 Disallow: / User-agent: Test Spider Disallow: / User-agent: dotbot Disallow: /