### /robots.txt file # if not present or if empty, all robots will consider themselves welcome. # Place exceptions first and the catch all last. # There is no Allow field. Only Disallow. # http://www.robotstxt.org/ # # $Id:$ # User-agent: * Crawl-delay: 11 Disallow: /cgi-bin/ #Disallow: /event/ Disallow: /event?category ## To allow a single robot ## User-agent: WebCrawler ## Disallow: # To exclude all other robots from the entire server ## User-agent: * ## Disallow: /