# 
# Agent-specific settings for indexing the site.
#
# 
# Generic settings for indexing the site.
#
User-agent: *
Sitemap: http://www.oecd-ilibrary.org/content/images/sitemap/index.xml
Crawl-delay: 10
Disallow: /search
Disallow: /session
Disallow: /admin/
Disallow: /counterstats/
Disallow: /regactivation
Disallow: /registration

# Specific crawler settings - as per request RT-14813
User-agent:AhrefsBot
Disallow: /