User-Agent: ia_archiver # Amazon Alexa Crawler Allow: / Disallow: /admin User-Agent: DuckDuckBot Allow: / Disallow: /admin User-Agent: Slurp # Yahoo Allow: / Disallow: /admin User-Agent: Bingbot Allow: / Disallow: /admin User-Agent: Applebot Allow: / Disallow: /admin User-Agent: Googlebot Allow: / Disallow: /admin User-Agent: Googlebot-News Allow: / Disallow: /admin User-Agent: Googlebot-Image/1.0 Allow: / Disallow: /admin User-Agent: Googlebot-Video/1.0 Allow: / Disallow: /admin User-Agent: Facebot # Facebook Allow: / Disallow: /admin User-Agent: facebookexternalhit/1.1 # Facebook Allow: / Disallow: /admin User-Agent: * Disallow: /