# Enable the following if needed by removing the # sign at the start of the appropriate lines

# The W3C link checker
User-Agent: W3C-checklink
Disallow:

# A powerful full site html validator
User-agent: WDG_SiteValidator
Disallow:


# This should stay in place
User-agent: *
Disallow: /cgi-bin/

#  Usually it may be preferable to block these fully but you decide

#User-agent: Googlebot-Image
#Disallow: / 

#User-agent: searchpreview
#Disallow: /

#User-agent: gigabot
#Disallow: /