User-Agent: * Disallow: /admin/ Disallow: /search/ # # Disallow for WebResource.axd caching issues. Several instances below to cover all search engines. # # To specify matching the end of a URL, use $ # Disallow: /*.axd$ # # However, WebResource.axd and ScriptResource.axd always include a query string parameter the URL does # not end with .axd thus, the correct robots.txt record for Google would be: # Disallow: /*.axd # # Not all crawlers recognize the wildcard '*' syntax. To comply with the robots.txt draft RFC # Note that the records are case sensitive, and error page is showing the requests to be in lower case # so let's include both cases below: # Disallow: /ScriptResource.axd Disallow: /WebResource.axd Disallow: /ImageGenerator.axd Disallow: /DESGetFiles.aspx Sitemap: http://orecorp.com.au/sitemap.xml User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: WebBandit Disallow: / User-agent: WebZIP Disallow: / User-agent: WebReaper Disallow: / User-agent: WebStripper Disallow: / User-agent: Web Downloader Disallow: / User-agent: WebCopier Disallow: / User-agent: Offline Explorer Pro Disallow: / User-agent: HTTrack Website Copier Disallow: / User-agent: Offline Commander Disallow: / User-agent: Leech Disallow: / User-agent: WebSnake Disallow: / User-agent: BlackWidow Disallow: / User-agent: HTTP Weazel Disallow: /