User-agent: * # match all bots Crawl-delay: 10 Disallow: /cgi-bin/ # keep them out Disallow: /tmp/ Request-rate: 1/5 # maximum rate is one page every 5 seconds Visit-time: 0600-0845 # only visit between 06:00 and 08:45 UTC (GMT)