User-agent: slurp
Disallow: /custom/
Disallow: /admin/
Disallow: /get/
Disallow: /scripts/
Disallow: /styles/
Disallow: /images/
Disallow: /custom_images/
Disallow: /subscribe/
Disallow: /api/
Allow: /
Crawl-delay: 1000

User-agent: *
Disallow: /custom/
Disallow: /admin/
Disallow: /get/
Disallow: /scripts/
Disallow: /styles/
Disallow: /images/
Disallow: /custom_images/
Disallow: /subscribe/
Disallow: /api/
Allow: