User-agent: IRLbot Crawl-delay: 600 Disallow: /test/ Disallow: /phpBB2/admin/ Disallow: /phpBB2/memberlist.php Disallow: /phpBB2/ctracker/ Disallow: /pages/1teenieclubpag.html User-agent: wget Crawl-delay: 300 Disallow: /test/ Disallow: /phpBB2/admin/ Disallow: /phpBB2/memberlist.php Disallow: /phpBB2/ctracker/ Disallow: /pages/1teenieclubpag.html # # Doesn't follow robots.txt anyway, but... # User-agent: k2spider Disallow: / User-agent: NPBot Disallow: / # A capture bot, downloads gazillions of pages with no public benefit # http://www.webreaper.net/ User-agent: WebReaper Disallow: / #rest of bots user-agent: * Disallow: /test/ Disallow: /phpBB2/admin/ Disallow: /phpBB2/memberlist.php Disallow: /phpBB2/ctracker/ Disallow: /pages/1teenieclubpag.html Allow: /cpg1412/thumbnails.php?album=topn&cat=0&page=1 Allow: /cpg1412/thumbnails.php?album=topn&cat=0&page=2