User-Agent: * # directed to all robots
Crawl-delay: 100
Disallow: /
Disallow: /icons
Disallow: /pub
Disallow: /images
Disallow: /stats
Disallow: /awstats-icon
Disallow: /munin
Disallow: /media