# $Id: robots.txt,v 1.9.2.1 2008/12/10 20:12:19 goba Exp $
#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html

User-agent: *
Crawl-delay: 10
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /sites/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /contact/
Disallow: /logout/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=contact/
Disallow: /?q=logout/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
# added on 2009-10-22
Disallow: /display.image
Disallow: /article.email
Disallow: /display.article
Disallow: /article.print
Disallow: /display.image/
Disallow: /article.email/
Disallow: /display.article/
Disallow: /article.print/
Disallow: /yaleglobal.yale.edu/
Disallow: /error.php
# added on 2010-04-02
Disallow: /article_list/1
Disallow: /article_list/2
Disallow: /article_list/3
Disallow: /article_list/4
Disallow: /article_list/5
Disallow: /article_list/6
Disallow: /article_list/7
Disallow: /article_list/8
Disallow: /article_list/9
Disallow: /article_list/10
Disallow: /article_list/11
Disallow: /article_list/12
Disallow: /article_list/13
Disallow: /article_list/14
Disallow: /article_list/15
Disallow: /article_list/16
Disallow: /article_list/17
Disallow: /article_list/18
Disallow: /article_list/19
Disallow: /archive/200101
Disallow: /archive/200102
Disallow: /archive/200103
Disallow: /archive/200104
Disallow: /archive/200105
Disallow: /archive/200106
Disallow: /archive/200107
Disallow: /archive/200108
Disallow: /archive/200109
Disallow: /archive/200110
Disallow: /archive/200111
Disallow: /archive/200112
Disallow: /archive/200201
Disallow: /archive/200202
Disallow: /archive/200203
Disallow: /archive/200204
Disallow: /archive/200205
Disallow: /archive/200206
Disallow: /archive/200207
Disallow: /archive/200208
Disallow: /archive/200209
Disallow: /archive/200210
Disallow: /archive/200211
Disallow: /archive/200212
Disallow: /archive/200301
Disallow: /archive/200302
Disallow: /archive/200303
Disallow: /archive/200304
Disallow: /archive/200305
Disallow: /archive/200306
Disallow: /archive/200307
Disallow: /archive/200308
Disallow: /archive/200309
Disallow: /archive/200310
Disallow: /archive/200311
Disallow: /archive/200312
Disallow: /archive/200401
Disallow: /archive/200402
Disallow: /archive/200403
Disallow: /archive/200404
Disallow: /archive/200405
Disallow: /archive/200406
Disallow: /archive/200407
Disallow: /archive/200408
Disallow: /archive/200409
Disallow: /archive/200410
Disallow: /archive/200411
Disallow: /archive/200412
Disallow: /archive/200501
Disallow: /archive/200502
Disallow: /archive/200503
Disallow: /archive/200504
Disallow: /archive/200505
Disallow: /archive/200506
Disallow: /archive/200507
Disallow: /archive/200508
Disallow: /archive/200509
Disallow: /archive/200510
Disallow: /archive/200511
Disallow: /archive/200512
Disallow: /archive/200601
Disallow: /archive/200602
Disallow: /archive/200603
Disallow: /archive/200604
Disallow: /archive/200605
Disallow: /archive/200606
Disallow: /archive/200607
Disallow: /archive/200608
Disallow: /archive/200609
Disallow: /archive/200610
Disallow: /archive/200611
Disallow: /archive/200612
Disallow: /archive/200701
Disallow: /archive/200702
Disallow: /archive/200703
Disallow: /archive/200704
Disallow: /archive/200705
Disallow: /archive/200706
Disallow: /archive/200707
Disallow: /archive/200708
Disallow: /archive/200709
Disallow: /archive/200710
Disallow: /archive/200711
Disallow: /archive/200712
Disallow: /archive/200801
Disallow: /archive/200802
Disallow: /archive/200803
Disallow: /archive/200804
Disallow: /archive/200805
Disallow: /archive/200806
Disallow: /archive/200807
Disallow: /archive/200808
Disallow: /archive/200809
Disallow: /archive/200810
Disallow: /archive/200811
Disallow: /archive/200812
Disallow: /archive/200901
Disallow: /archive/200902
Disallow: /archive/200903
Disallow: /archive/200904
Disallow: /archive/200905
Disallow: /archive/200906
Disallow: /archive/200907
Disallow: /archive/200908
Disallow: /archive/200909
Disallow: /archive/200910
Disallow: /archive/200912
Disallow: /archive/201001
Disallow: /archive/201002
Disallow: /archive/201003
Disallow: /archive/201004
# added on 6/11/2010
Disallow: /en
Disallow: /en/
Disallow: /node
Disallow: /node/
Disallow: /cn/node
Disallow: /cn/node/
# added on 6/12/2010
Disallow: /bound_together 
Disallow: /book_reviews
Disallow: /book_excerpts
Disallow: /special_report/

Disallow: /cn/bound_together 
Disallow: /cn/book_reviews
Disallow: /cn/book_excerpts
Disallow: /cn/special_report/

Disallow: /rss_twitter
Disallow: /feeds/