User-agent: *
Disallow: /Admin
Disallow: /includes
Disallow: /javascript

User-agent: TwengaBot
Crawl-Delay: 500

User-agent: psbot
Disallow: /
User-Agent: searchdnabot
Crawl-Delay:   120
User-Agent: MJ12bot
Crawl-Delay:   120

User-agent: ShopWiki
Crawl-delay: 10

User-agent: Yandex
Disallow: /
User-agent: Twiceler
Crawl-delay: 40
User-agent: Purebot
Crawl-delay: 400

User-agent: zermelo
Crawl-delay:300

User-agent: WebDataCentreBot
Crawl-delay: 1000

User-agent: WebDataCentreBot/1.0
Crawl-delay: 1000

User-agent: powerset
Crawl-delay: 300
User-agent: Slurp
Crawl-delay: 30


User-agent: Accelobot
Crawl-delay: 30


User-agent: discobot/1.0
Crawl-Delay: 2000

User-agent: voyager
Crawl-Delay: 60

User-agent: TurnitinBot
Disallow: / 
Crawl-Delay: 60000

User-agent: noxtrum
Disallow: / 

User-agent: IRLbot 
Disallow: /
Crawl-Delay: 60000


User-agent: ia_archiver
Crawl-Delay: 120

User-agent: Ocelli
Crawl-delay: 2000
User-agent: Speedy
Crawl-Delay: 50

User-agent: panscient
Crawl-Delay: 45

User-agent: panscient.com
Crawl-Delay: 45

User-agent: Baiduspider+
Disallow: /
Crawl-Delay: 9945

User-agent: dotbot
Crawl-Delay: 45
User-agent: Baiduspider
Disallow: /
Crawl-Delay: 9945