User-agent: * # All user-agents (except others specified in this file) Disallow: /cgi-bin/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ Disallow: /Analog/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ Disallow: /Archived%20Logs/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ Disallow: /Backup/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ Disallow: /Backup%20Settings/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ Disallow: /CSAD%20Maintenance/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ Disallow: /Plug-Ins/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ Disallow: /Teaching/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ Disallow: /Search/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ Disallow: /Texts/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ Disallow: /Tractatus/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ Disallow: /cgi-bin/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ Disallow: /cgi-bin/ # Dont look at stuff in http://my.site.ac.uk/cgi-bin/ User-agent: WebWatch/v3.0 Disallow: # The WebWatch robot is allowed to look at everything :-)