# robots.txt for http://www.ams.org/
Sitemap: http://www.ams.org/sitemap.xml
Sitemap: http://www.ams.org/sitemap_news.xml
Sitemap: http://www.ams.org/sitemapindex.xml

# a list of URL directories/files for wandering robots to ignore.
#
# User-agent contains entries of robots that you want to match this
# against
#
# * User-agent indicates all robots excluded from the following URL(s)
#
User-agent: AdsBot-Google 	
Crawl-Delay: 10
Disallow: /
User-agent: Slurp
Crawl-Delay: 10
Disallow: /mathscinet-getitem/
User-agent: *
Crawl-Delay: 10
User-agent: *
#
# Allow Google to get our news RSS feed
# added by txm 6 july 2011
User-agent: Googlebot
Allow: /cgi-bin/content/news_items.cgi?rss=1

# Stops Google's Image bot from crawling our site's images and 
# making them searchable online, if just to save bandwidth
# added by txm 6 july 2011

User-agent: Googlebot-Image
Disallow: /images/

#Baiduspider
User-agent: Baiduspider
Disallow: /

# unpublished papers reside here
#
#Disallow: /journals/submitted/
Disallow: /distribution/
Disallow: /editflow/ 
#Disallow: /journals/submission/logs/
#Disallow: /abstracts/logs/
Disallow: /ams/press/
Disallow: /leavingmsn/
Disallow: /msnpdf/
Disallow: /mathscinet-docdelivery/
Disallow: /mathscinet-relay/
Disallow: /mathscinet/
Disallow: /mathscinet/addclip.html
Disallow: /msnhtml/
Disallow: /mathscinet-getauth
Disallow: /mref/
Disallow: /batchmref/
Disallow: /vendor-info/
Disallow: /bboards/
Disallow: /online_bks/
Disallow: /msdemo/
Disallow: /peerdomo/
Disallow: /cbms/cbms2010-work/
Disallow: /cgi-bin/msdemo/
Disallow: /cgi-bin/peerdemo/
Disallow: /peertrack/
Disallow: /ams/web/peer_review/bull/
Disallow: /ecbt/
Disallow: /treas-reports/
Disallow: /msnmain
Disallow: /mrlookup/
Disallow: /batchmrlookup/
## We will allow /cgi-bin/bookstore so that Google can index the bookstore
##Disallow: /cgi-bin/bookstore/
Disallow: /ams/staffdirectory.html
Disallow: /investcom/
Disallow: /prof-services/
Disallow: /cgi-bin
##Disallow: /bookstore/
##Disallow: /bookpromo/
##Disallow: /booksearch
Disallow: /bookexpress
##Disallow: /bookstore-getitem
Disallow: /amsbookstore
Disallow: /amsshop
Disallow: /cml/
Disallow: /cml-lookup/
Disallow: /jour-getitem/
Disallow: /journal-getitem/
Disallow: /delete/
Disallow: /joursearch/
Disallow: /STIX/
Disallow: /STIX/images/
Disallow: /STIX/ion/Univ/
Disallow: /ebc-private/bboard2002/
## We will allow /jourcgi so that Google can get lists of issues
##Disallow: /jourcgi
Disallow: /peer-review/
Disallow: /mresub/
Disallow: /submit-book-journal/
Disallow: /mathscinet-getitem/
Disallow: /amsmtgs/2124_abstracts/1056-30-23.pdf
Disallow: /meetings/national/jmm/2124_abstracts/1056-30-23.pdf
Disallow: /profession/data/annual-survey/AS-Committee/
Disallow: /prof-services/nsa/
Disallow: /testing/