# Robots policy for irg-wp.com

# Dynamic search results and the members area are not useful content for
# search indexes, and crawling paginated search results generates heavy
# database load. Document detail pages (/irgdocs/details.php) remain
# crawlable — they carry the indexable content.
User-agent: *
Disallow: /irgdocs/search.php
Disallow: /account/
Disallow: /account_test/
Crawl-delay: 10

# Abusive crawlers — block outright.
User-agent: SleepBot
Disallow: /

User-agent: Bytespider
Disallow: /