# Robots.txt file from http://www.searchengineworld.com # # Bans from cgi-bin and images # # Disallow /cgi-bin/ User-agent: * Disallow: /cgi-bin/ Disallow: /css/ Disallow: /js/ Disallow: /ar/ Disallow: /zodiac/ Disallow: /govlinks/ Disallow: /greybox/ Disallow: /images/ Disallow: /k12/ Disallow: /support/ Disallow: /httpdocs/ad_images Disallow: /httpdocs/.htaccess Disallow: /httpdocs/400.shtml Disallow: /httpdocs/401.shtml Disallow: /httpdocs/403.shtml Disallow: /httpdocs/404.shtml Disallow: /httpdocs/500.php Disallow: /httpdocs/500.shtml Disallow: /httpdocs/ctreturn.html Disallow: /httpdocs/default.html Disallow: /httpdocs/favicon.ico Disallow: /httpdocs/fbackreturn.html Disallow: /httpdocs/feed.rss Disallow: /httpdocs/headlines.html Disallow: /httpdocs/rcontactus.html Disallow: /httpdocs/tag_cloud.html