# robots.txt for http://www.ht.com.au

# Exclusions for all Robots

User-agent: *
Disallow:
Disallow: /attach/
Disallow: /common/
Disallow: /email/
Disallow: /error/
Disallow: /images/
Disallow: /information/
Disallow: /javascript/
Disallow: /stylesheets/
Disallow: /popup/true/
Disallow: /delivery_popup.hts
Disallow: /account.hts
Disallow: /order_list.hts
Disallow: /order_detail.hts
Disallow: /address_list.hts
Disallow: /address_detail.hts
Disallow: /subscriptions.hts
Disallow: /contact_information.hts
Disallow: /payment_pref.hts
Disallow: /cart.hts
Disallow: /addcart.hts
Disallow: /checkout.hts
Disallow: /setsort.hts
Disallow: */page/*/list.hts
Disallow: */listview/*/list.hts
Disallow: /group.hts
Disallow: /area.hts

# Robots to completely disallowed

User-agent: *libwww* 
Disallow: /

User-agent: HttpTool* 
Disallow: /

User-agent: *Fetch* 
Disallow: /

User-agent: http://www.almaden.ibm.com/cs/crawler 
Disallow: /

User-agent: BorderManager*
Disallow: /

User-agent: Maxthon
Disallow: /

User-agent: Bilbo/2.3b-UNIX
Disallow: /

User-agent: webcollage/1.77
Disallow: /

User-agent: FAST-WebCrawler/3.6/FirstPage
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: Java*
Disallow: /

User-agent: Offline Explorer/1.9
Disallow: /

User-agent: Missigua Locator 1.9
Disallow: /

User-agent: grub-client 
Disallow: /

User-agent: msrbot
Disallow: /

User-agent: lwp*
Disallow: /

User-agent: *newsexpress*
Disallow: /

User-agent: ShopWiki
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: rogerbot 
Disallow: /  

User-agent: Yandex 
Disallow: / 

User-agent: Googlebot
Allow: /attach/
Disallow:
Disallow: /common/
Disallow: /email/
Disallow: /error/
Disallow: /images/
Disallow: /information/
Disallow: /javascript/
Disallow: /stylesheets/
Disallow: /popup/true/
Disallow: /delivery_popup.hts
Disallow: /account.hts
Disallow: /order_list.hts
Disallow: /order_detail.hts
Disallow: /address_list.hts
Disallow: /address_detail.hts
Disallow: /subscriptions.hts
Disallow: /contact_information.hts
Disallow: /payment_pref.hts
Disallow: /cart.hts
Disallow: /addcart.hts
Disallow: /checkout.hts
Disallow: /setsort.hts
Disallow: */page/*/list.hts
Disallow: */listview/*/list.hts
Disallow: /group.hts
Disallow: /area.hts

User-agent: speedy
Crawl-Delay: 86400

User-agent: twenga
Crawl-Delay: 86400