# This is for every spider!
User-Agent: *
# stay away from this
Disallow: /cgi
Disallow: /js
Disallow: /javascript
Disallow: /servlets
Disallow: /servlet
Disallow: /jservlet
Disallow: /logs
Disallow: /stats_data
Disallow: /icon
Disallow: /icons
Disallow: /FCKeditor
Disallow: /forms
Disallow: /reports
Disallow: /cronjobs
Disallow: /dev
Disallow: /etc
Disallow: /usr
Disallow: /bin
Disallow: /pub
Disallow: /classes
Disallow: /css
Crawl-delay: 14
Disallow: #This could be habit forming!
# Don't comments make code much more readable!!!