# robots.txt for http://www.cs.inf.ethz.ch/ User-agent: MOMspider # The Multi-Owner Maintenance Spider Disallow: /cs/group/gross/local/ Disallow: /cgi-bin/ # Script files Disallow: /MOMspider/ # local MOMspider output User-agent: * # all other spiders should avoid Disallow: /cs/group/gross/local/ Disallow: /cgi-bin/ # Script files Disallow: /MOMspider/ # local MOMspider output Disallow: /afs/