# For Mediapartners-Google, Googlebot, and Slurp set the crawl delay to 1 second
User-agent: Mediapartners-Google
Disallow: /r/candidates/
Disallow: /c/companies/
Disallow: /btrp/
Disallow: /r/jobs/applicant.cfm
Crawl-delay: 1

User-agent: Googlebot
Disallow: /r/candidates/
Disallow: /c/companies/
Disallow: /btrp/
Disallow: /r/jobs/applicant.cfm
Crawl-delay: 1

User-agent: Slurp
Disallow: /r/candidates/
Disallow: /c/companies/
Disallow: /btrp/
Disallow: /r/jobs/applicant.cfm
Crawl-delay: 1

# Bad Bots:
# gsa-crawler, and MJ12bot - Disallow from root 
User-agent: gsa-crawler
Disallow: /

User-agent: MJ12bot
Disallow: /

# For all user-agent types, disallow candidates, companies, and bot-trap directories.
# Force the crawl delay for all bots to be one hour
User-agent: *
Disallow: /r/candidates/
Disallow: /c/companies/
Disallow: /btrp/
Disallow: /r/jobs/applicant.cfm
Crawl-delay: 3600