# robots.txt for http://arXiv.org/ and mirror sites http://*.arXiv.org/

User-agent: *
Disallow: /cgi-bin/
Disallow: /ad/

User-agent: Googlebot
Allow: /archive
Allow: /year
Allow: /list
Allow: /abs
Allow: /pdf
Allow: /html
Allow: /catchup
Disallow: /cgi-bin/

User-agent: Yahoo! Slurp
Crawl-delay: 20
Allow: /archive
Allow: /year
Allow: /list
Allow: /abs
Disallow: /cgi-bin/