User-agent: * Disallow: /rhyme/*/*/*/all/*/ Crawl-delay: 10 # Hits many times per second, not acceptable # http://www.nameprotect.com/botinfo.html User-agent: NPBot Disallow: / # A capture bot, downloads gazillions of pages with no public benefit # http://www.webreaper.net/ User-agent: WebReaper Disallow: / User-agent: Mediapartners-Google Allow: * # BEGIN XML-SITEMAP-PLUGIN Sitemap: http://www.rhymebox.de/sitemap.xml.gz # END XML-SITEMAP-PLUGIN