# Directions for robots.  See this URL:
# http://info.webcrawler.com/mak/projects/robots/norobots.html
# for a description of the file format.
# 2008-08-21
#####
# Here is where we override the default action

## Due to a bug in linklint, must first specify a disallow in order for 
## for all other directories to be allowed. Feel free to add other
## disallows below the first disallow line.

User-agent: LinkLint
Disallow: /workaroundForLinkLintRandomDirForConfig/ 

#####
# Allow W3C link Validator for /dev/ and /nisdev/
# skipping other dynamic content or private areas
# 2004-08-27 gaudette
#
User-agent: W3C-checklink
Disallow: /cms/
Disallow: /cgi-bin/
Disallow: /htbin/
Disallow: /htbin.ph/
Disallow: /BUbin/
Disallow: /bubin/
Disallow: /testing/
Disallow: /TESTING/
Disallow: /IT/SoftwareDist/
Disallow: /it/SoftwareDist/
Disallow: /software/
Disallow: /SOFTWARE/
Disallow: /IT/new/
Disallow: /it/new/
Disallow: /nis/
Disallow: /nishd/
Disallow: /library/working/
Disallow: /library/WORKING/
Disallow: /reports/
Disallow: /bulletins/work/
Disallow: /admissions/test/
Disallow: /cas/oldsite/
Disallow: /MPA/
Disallow: /finaid/test/
Disallow: /naitest/
Disallow: /newswire/
Disallow: /practice/
Disallow: /providers/
Disallow: /stats/
Disallow: /usc/test/
Disallow: /webcentral/output/
Disallow: /webmail/
Disallow: /alumni/portfolio/
Disallow: /dev/

#####
# default action - currently it allows access to most of the site
# skipping dynamic content or private areas
#
User-agent: gsa-crawler
Disallow: /cgi-bin/
Disallow: /cms/
Disallow: /htbin/
Disallow: /htbin.ph/
Disallow: /BUbin/
Disallow: /bubin/
Disallow: /testing/
Disallow: /TESTING/
Disallow: /IT/SoftwareDist/
Disallow: /it/SoftwareDist/
Disallow: /software/
Disallow: /SOFTWARE/
Disallow: /IT/new/
Disallow: /it/new/
Disallow: /library/working/
Disallow: /library/WORKING/
Disallow: /reports/
Disallow: /nisdev/
Disallow: /bulletins/work/
Disallow: /admissions/test/
Disallow: /cas/oldsite/
Disallow: /MPA/
Disallow: /finaid/test/
Disallow: /naitest/
Disallow: /newswire/
Disallow: /practice/
Disallow: /providers/
Disallow: /stats/
Disallow: /usc/test/
Disallow: /webcentral/output/
Disallow: /webmail/
Disallow: /alumni/portfolio/
Disallow: /dbin/dos/ocs/
Disallow: /dev/
Disallow: /wbur/arts/
Disallow: /wbur/connection/
Disallow: /wbur/herenow/
Disallow: /wbur/livingonearth/
Disallow: /wbur/miscellaneous/
Disallow: /wbur/onpoint/
Disallow: /wbur/special_projects_unit/
Disallow: /wbur/wburnews/
Disallow: /wbur/woi/
Disallow: /link/
Disallow: /home-media/

#####
# default action - currently it allows access to most of the site
# skipping dynamic content or private areas
#
User-agent: *
Disallow: /cms/
Disallow: /cgi-bin/
Disallow: /htbin/
Disallow: /htbin.ph/
Disallow: /BUbin/
Disallow: /bubin/
Disallow: /testing/
Disallow: /TESTING/
Disallow: /IT/SoftwareDist/
Disallow: /it/SoftwareDist/
Disallow: /software/
Disallow: /SOFTWARE/
Disallow: /IT/new/
Disallow: /it/new/
Disallow: /nis/
Disallow: /library/working/
Disallow: /library/WORKING/
Disallow: /reports/
Disallow: /nisdev/
Disallow: /bulletins/work/
Disallow: /admissions/test/
Disallow: /cas/oldsite/
Disallow: /MPA/
Disallow: /finaid/test/
Disallow: /naitest/
Disallow: /newswire/
Disallow: /practice/
Disallow: /providers/
Disallow: /stats/
Disallow: /usc/test/
Disallow: /webcentral/output/
Disallow: /webmail/
Disallow: /alumni/portfolio/
Disallow: /dbin/dos/ocs/
Disallow: /dev/
Disallow: /wbur/arts/
Disallow: /wbur/connection/
Disallow: /wbur/herenow/
Disallow: /wbur/livingonearth/
Disallow: /wbur/miscellaneous/
Disallow: /wbur/onpoint/
Disallow: /wbur/special_projects_unit/
Disallow: /wbur/wburnews/
Disallow: /wbur/woi/
Disallow: /link/
Disallow: /home-media/
Crawl-delay: 15

# If your robot is polite enough to request only a small number of
# files at a time we will add your robot to the list of cool robots.
# Send email to webmaster@bu.edu, referring to the /robots.txt
# URL.