# robots.txt file for athenus.com - The Engineering and Science Search Engine

# Let Google Adsense crawl everything for best targeting 
User-agent: Mediapartners-Google*
Disallow: /stats/

# Block OmniExplorer
USer-agent: OmniExplorer_Bot
Disallow: /
User-agent: OmniExplorer_Bot*
Disallow: /
Disallow: *
User-agent: OmniExplorer_Bot/1.09
Disallow: /
Disallow: *
User-agent: OmniExplorer_Bot/1.07
Disallow: /
Disallow: *

# Ok, now general blocking. As of April 22nd 2005 we are blocking our search results from bots.
# Prevents bandwidth stealing for starters and probably crashing of nice bots.
User-agent: *
Disallow: /cgi-bin/
Disallow: /stats/

# Block our agent from crawling our search results (Failsafe)
User-agent: Athenusbot
Disallow: /cgi-bin/
Disallow: /stats/