User-agent: *
Disallow: /cgi-bin/
User-agent: Slurp
Crawl-delay: 20
User-agent: Exabot
Disallow: /shop/
User-agent: Yandex
Disallow: /
User-agent: Baidu
Disallow: /
User-agent: Baiduspider
Disallow: /