# Disallow all crawlers access to certain pages.
User-agent: *
Disallow: /common/
Disallow: /SD/
Disallow: /D/
Disallow: /Online-Degree/
Disallow: /speedydegrees/
Disallow: /cgi-bin/
Disallow: /inner/
Disallow: /edu/CustFilesFlaxible/
Disallow: /speedydegrees/policies.asp
Disallow: /speedydegrees/pp.asp


# Disallow crawlers access site.
User-agent: ia_archiver-web.archive.org
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: ia_archiver-web.archive.org
Disallow: /robots.txt

User-agent: ia_archiver-web.archive.org
Disallow: robots.txt


User-agent: ia_archiver
Disallow: /robots.txt

User-agent: ia_archiver
Disallow: robots.txt

User-agent: AboutUsBot 
Disallow: /