User-agent: *
Sitemap: /sitemap.php
Disallow: /cgi-bin
Disallow: /cgi-perl
Disallow: /mpapps
Disallow: /mpsearch
Disallow: /news.php
Disallow: /extlink.php
Disallow: /regionselect.php
Disallow: /tweetstatus.php

User-agent: StockCrawler
Disallow: /