#sitemap of RTA website to point spiders to this page for indexing
Sitemap: http://webuat-whome.rta.nsw.gov.au/sitemap.xml

# These directives are for bad robots.
# Directoves here ask the robots not to index the entire site
# New robots can  be added using the following example
# User-agent: bad-robot
# Disallow: /
 
# For all other spiders not filtered out above
# Common directories that do not need to be indexed
# Additional directories can be added using "Disallow: /xxx". 
# Generally regular expression or expansions are not supported in directories
User-agent: *
Disallow: /sitesearch/
#Crawl-delay: 5

# Additional data in the event that government spiders are created 
# User-agent: AGLS
# Meta: /meta/index.xml