# /robots.txt file for http://www.fpbooks.com # right now, we are allowing spidering # if we want to disallow it, we can uncomment the following lines. #User-agent: * #Disallow: / #the following was really hammering us #User-agent: ia_archiver #Disallow: / #User-agent: PortalBSpider #Disallow: / #User-agent: Slurp #Crawl-delay: 10 # all robots are welcome #User-agent: * #Disallow: redirect.asp User-agent: Slurp Crawl-delay: 10 # all robots are welcome User-agent: *