User-agent: *
Crawl-delay: 20

User-agent: Mediapartners-Google
Disallow:

User-agent: Googlebot
Disallow:

User-agent: Adsbot-Google
Disallow:

User-agent: Googlebot-Image
Disallow:

User-agent: Googlebot-Mobile
Disallow:

User-agent: DuckDuckBot
Disallow:

User-agent: facebot
Disallow:

User-agent: ia_archiver
Disallow:

User-agent: APIs-Google
Disallow:

User-agent: Googlebot-News
Disallow:

User-agent: AdsBot-Google-Mobile
Disallow:

User-agent: MSNBot
Disallow:

User-agent: bingbot
Disallow:

User-agent: Slurp
Disallow:

User-agent: Yahoo! Slurp
Disallow:

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: WeSEE
Disallow: /

User-agent: proximic
Disallow: /

# Block all other spiders
User-agent: *
Disallow: /

# Block Directories for all spiders
User-agent: *
Disallow: /admin/
Disallow: /test/