# robots.txt for http://www.brianswilson.org/ # This file will assist portals in indexing the website. User-agent: * Disallow: /cgi-bin/ # These are my perl scripts. Disallow: /logs/ # Nothing in this directory to be indexed. Disallow: /docs/art/ # No html files in here to be indexed. Disallow: /docs/sound/ # No html files in here to be indexed.