# Robots.txt # for more information using this: # http://www.robotstxt.org/ # Adapted for msa-ast.org 3/8/2008 User-agent: * # dont need to crawl the Frontpage system folders Disallow: /_fpclass Disallow: /_overlay Disallow: /_private Disallow: /_themes # dont need to crawl javascript Disallow: /cgi Disallow: /Scripts Disallow: /javascript # let the images be crawled (for image search) # remove hash if you dont want them to be # Disallow: /images # Misc folders Disallow: /Newsite Disallow: /Fall07 Disallow: /Workshops_files Disallow: /resource files Disallow: /delegate-info_files