#****************************************************************************
# robots.txt
#     : Robots, spiders, and search engines use this file to detmine which 
#       content they should *not* crawl while indexing your website.
#     : This system is called "The Robots Exclusion Standard."
#     : It is strongly encouraged to use a robots.txt validator to check
#       for valid syntax before any robots read it!
#
# Examples:
#
# Instruct all robots to stay out of the admin area.
#     : User-agent: *
#     : Disallow:   /admin/
#
# Restrict Google and MSN from indexing your images.
#     : User-agent: Googlebot
#     : Disallow:   /images/
#     : User-agent: MSNBot
#     : Disallow:   /images/
#****************************************************************************

    User-agent: Googlebot
    Crawl-delay: 10

    Allow: /site/robots.txt
    Allow: /site/index.php

    # Directories
    Disallow: /tracker/
    Disallow: /xtracker/
    Disallow: /user/
    Disallow: /book/export/
    Disallow: /forward/
    Disallow: /database/
    Disallow: /includes/
    Disallow: /misc/
    Disallow: /modules/
    Disallow: /sites/
    Disallow: /themes/
    Disallow: /scripts/
    Disallow: /updates/
    Disallow: /profiles/
    Disallow: /comment/
    Disallow: /taxonomy/
    Disallow: /popular/
    Disallow: */feed*
    Disallow: */comment/reply/
    Disallow: /popular/*
    Disallow: /comments*
    Disallow: /frontpage*
    Disallow: /comments*
    Disallow: /aggregator*
    Disallow: /aggregator2*
    Disallow: *?sort=asc&order=Time*
    Disallow: *comment-*
    # Files
    Disallow: /rss.xml
    Disallow: /feed
    Disallow: /xmlrpc.php
    Disallow: /cron.php
    Disallow: /update.php
    Disallow: /install.php
    Disallow: /INSTALL.mysql.txt
    Disallow: /INSTALL.pgsql.txt
    Disallow: /CHANGELOG.txt
    Disallow: /MAINTAINERS.txt
    Disallow: /LICENSE.txt
    Disallow: /UPGRADE.txt

    # Paths (clean URLs)
    Disallow: /admin/
    Disallow: /comment/reply/
    Disallow: /contact/
    Disallow: /logout/
    Disallow: /node/add/
    Disallow: /search/
    Disallow: /user/register/
    Disallow: /user/password/
    Disallow: /user/login/

    # Paths (no clean URLs)
    Disallow: /?q=admin/
    Disallow: /?q=comment/reply/
    Disallow: /?q=contact/
    Disallow: /?q=logout/
    Disallow: /?q=node/add/
    Disallow: /?q=search/
    Disallow: /?q=user/password/
    Disallow: /?q=user/register/
    Disallow: /?q=user/login/
    
User-agent: MSNBot
    Crawl-delay: 10

    Allow: /site/robots.txt
    Allow: /site/index.php

    # Directories
    Disallow: /tracker/
    Disallow: /xtracker/
    Disallow: /user/
    Disallow: /book/export/
    Disallow: /forward/
    Disallow: /database/
    Disallow: /includes/
    Disallow: /misc/
    Disallow: /modules/
    Disallow: /sites/
    Disallow: /themes/
    Disallow: /scripts/
    Disallow: /updates/
    Disallow: /profiles/
    Disallow: /comment/
    Disallow: /taxonomy/
    Disallow: /popular/
    Disallow: */feed*
    Disallow: */comment/reply/
    Disallow: /popular/*
    Disallow: /comments*
    Disallow: /frontpage*
    Disallow: /comments*
    Disallow: /aggregator*
    Disallow: /aggregator2*
    Disallow: *?sort=asc&order=Time*
    Disallow: *comment-*
    # Files
    Disallow: /rss.xml
    Disallow: /feed
    Disallow: /xmlrpc.php
    Disallow: /cron.php
    Disallow: /update.php
    Disallow: /install.php
    Disallow: /INSTALL.mysql.txt
    Disallow: /INSTALL.pgsql.txt
    Disallow: /CHANGELOG.txt
    Disallow: /MAINTAINERS.txt
    Disallow: /LICENSE.txt
    Disallow: /UPGRADE.txt

    # Paths (clean URLs)
    Disallow: /admin/
    Disallow: /comment/reply/
    Disallow: /contact/
    Disallow: /logout/
    Disallow: /node/add/
    Disallow: /search/
    Disallow: /user/register/
    Disallow: /user/password/
    Disallow: /user/login/

    # Paths (no clean URLs)
    Disallow: /?q=admin/
    Disallow: /?q=comment/reply/
    Disallow: /?q=contact/
    Disallow: /?q=logout/
    Disallow: /?q=node/add/
    Disallow: /?q=search/
    Disallow: /?q=user/password/
    Disallow: /?q=user/register/
    Disallow: /?q=user/login/
    
User-agent: Slurp
    Crawl-delay: 10

    Allow: /site/robots.txt
    Allow: /site/index.php

    # Directories
    Disallow: /tracker/
    Disallow: /xtracker/
    Disallow: /user/
    Disallow: /book/export/
    Disallow: /forward/
    Disallow: /database/
    Disallow: /includes/
    Disallow: /misc/
    Disallow: /modules/
    Disallow: /sites/
    Disallow: /themes/
    Disallow: /scripts/
    Disallow: /updates/
    Disallow: /profiles/
    Disallow: /comment/
    Disallow: /taxonomy/
    Disallow: /popular/
    Disallow: */feed*
    Disallow: */comment/reply/
    Disallow: /popular/*
    Disallow: /comments*
    Disallow: /frontpage*
    Disallow: /comments*
    Disallow: /aggregator*
    Disallow: /aggregator2*
    Disallow: *?sort=asc&order=Time*
    Disallow: *comment-*
    # Files
    Disallow: /rss.xml
    Disallow: /feed
    Disallow: /xmlrpc.php
    Disallow: /cron.php
    Disallow: /update.php
    Disallow: /install.php
    Disallow: /INSTALL.mysql.txt
    Disallow: /INSTALL.pgsql.txt
    Disallow: /CHANGELOG.txt
    Disallow: /MAINTAINERS.txt
    Disallow: /LICENSE.txt
    Disallow: /UPGRADE.txt

    # Paths (clean URLs)
    Disallow: /admin/
    Disallow: /comment/reply/
    Disallow: /contact/
    Disallow: /logout/
    Disallow: /node/add/
    Disallow: /search/
    Disallow: /user/register/
    Disallow: /user/password/
    Disallow: /user/login/

    # Paths (no clean URLs)
    Disallow: /?q=admin/
    Disallow: /?q=comment/reply/
    Disallow: /?q=contact/
    Disallow: /?q=logout/
    Disallow: /?q=node/add/
    Disallow: /?q=search/
    Disallow: /?q=user/password/
    Disallow: /?q=user/register/
    Disallow: /?q=user/login/
User-agent: Teoma
    Crawl-delay: 10

    Allow: /site/robots.txt
    Allow: /site/index.php

        # Directories
    Disallow: /tracker/
    Disallow: /xtracker/
    Disallow: /user/
    Disallow: /book/export/
    Disallow: /forward/
    Disallow: /database/
    Disallow: /includes/
    Disallow: /misc/
    Disallow: /modules/
    Disallow: /sites/
    Disallow: /themes/
    Disallow: /scripts/
    Disallow: /updates/
    Disallow: /profiles/
    Disallow: /comment/
    Disallow: /taxonomy/
    Disallow: /popular/
    Disallow: */feed*
    Disallow: */comment/reply/
    Disallow: /popular/*
    Disallow: /comments*
    Disallow: /frontpage*
    Disallow: /comments*
    Disallow: /aggregator*
    Disallow: /aggregator2*
    Disallow: *?sort=asc&order=Time*
    Disallow: *comment-*
    # Files
    Disallow: /rss.xml
    Disallow: /feed
    Disallow: /xmlrpc.php
    Disallow: /cron.php
    Disallow: /update.php
    Disallow: /install.php
    Disallow: /INSTALL.mysql.txt
    Disallow: /INSTALL.pgsql.txt
    Disallow: /CHANGELOG.txt
    Disallow: /MAINTAINERS.txt
    Disallow: /LICENSE.txt
    Disallow: /UPGRADE.txt

    # Paths (clean URLs)
    Disallow: /admin/
    Disallow: /comment/reply/
    Disallow: /contact/
    Disallow: /logout/
    Disallow: /node/add/
    Disallow: /search/
    Disallow: /user/register/
    Disallow: /user/password/
    Disallow: /user/login/

    # Paths (no clean URLs)
    Disallow: /?q=admin/
    Disallow: /?q=comment/reply/
    Disallow: /?q=contact/
    Disallow: /?q=logout/
    Disallow: /?q=node/add/
    Disallow: /?q=search/
    Disallow: /?q=user/password/
    Disallow: /?q=user/register/
    Disallow: /?q=user/login/

User-agent: Gigabot
    Crawl-delay: 10

    Allow: /site/robots.txt
    Allow: /site/index.php

    # Directories
    Disallow: /tracker/
    Disallow: /xtracker/
    Disallow: /user/
    Disallow: /book/export/
    Disallow: /forward/
    Disallow: /database/
    Disallow: /includes/
    Disallow: /misc/
    Disallow: /modules/
    Disallow: /sites/
    Disallow: /themes/
    Disallow: /scripts/
    Disallow: /updates/
    Disallow: /profiles/
    Disallow: /comment/
    Disallow: /taxonomy/
    Disallow: /popular/
    Disallow: */feed*
    Disallow: */comment/reply/
    Disallow: /popular/*
    Disallow: /comments*
    Disallow: /frontpage*
    Disallow: /comments*
    Disallow: /aggregator*
    Disallow: /aggregator2*
    Disallow: *?sort=asc&order=Time*
    Disallow: *comment-*
    # Files
    Disallow: /rss.xml
    Disallow: /feed
    Disallow: /xmlrpc.php
    Disallow: /cron.php
    Disallow: /update.php
    Disallow: /install.php
    Disallow: /INSTALL.mysql.txt
    Disallow: /INSTALL.pgsql.txt
    Disallow: /CHANGELOG.txt
    Disallow: /MAINTAINERS.txt
    Disallow: /LICENSE.txt
    Disallow: /UPGRADE.txt

    # Paths (clean URLs)
    Disallow: /admin/
    Disallow: /comment/reply/
    Disallow: /contact/
    Disallow: /logout/
    Disallow: /node/add/
    Disallow: /search/
    Disallow: /user/register/
    Disallow: /user/password/
    Disallow: /user/login/

    # Paths (no clean URLs)
    Disallow: /?q=admin/
    Disallow: /?q=comment/reply/
    Disallow: /?q=contact/
    Disallow: /?q=logout/
    Disallow: /?q=node/add/
    Disallow: /?q=search/
    Disallow: /?q=user/password/
    Disallow: /?q=user/register/
    Disallow: /?q=user/login/

User-agent: Scrubby
    Crawl-delay: 10

    Allow: /site/robots.txt
    Allow: /site/index.php

    # Directories
    Disallow: /tracker/
    Disallow: /xtracker/
    Disallow: /user/
    Disallow: /book/export/
    Disallow: /forward/
    Disallow: /database/
    Disallow: /includes/
    Disallow: /misc/
    Disallow: /modules/
    Disallow: /sites/
    Disallow: /themes/
    Disallow: /scripts/
    Disallow: /updates/
    Disallow: /profiles/
    Disallow: /comment/
    Disallow: /taxonomy/
    Disallow: /popular/
    Disallow: */feed*
    Disallow: */comment/reply/
    Disallow: /popular/*
    Disallow: /comments*
    Disallow: /frontpage*
    Disallow: /comments*
    Disallow: /aggregator*
    Disallow: /aggregator2*
    Disallow: *?sort=asc&order=Time*
    Disallow: *comment-*
    # Files
    Disallow: /rss.xml
    Disallow: /feed
    Disallow: /xmlrpc.php
    Disallow: /cron.php
    Disallow: /update.php
    Disallow: /install.php
    Disallow: /INSTALL.mysql.txt
    Disallow: /INSTALL.pgsql.txt
    Disallow: /CHANGELOG.txt
    Disallow: /MAINTAINERS.txt
    Disallow: /LICENSE.txt
    Disallow: /UPGRADE.txt

    # Paths (clean URLs)
    Disallow: /admin/
    Disallow: /comment/reply/
    Disallow: /contact/
    Disallow: /logout/
    Disallow: /node/add/
    Disallow: /search/
    Disallow: /user/register/
    Disallow: /user/password/
    Disallow: /user/login/

    # Paths (no clean URLs)
    Disallow: /?q=admin/
    Disallow: /?q=comment/reply/
    Disallow: /?q=contact/
    Disallow: /?q=logout/
    Disallow: /?q=node/add/
    Disallow: /?q=search/
    Disallow: /?q=user/password/
    Disallow: /?q=user/register/
    Disallow: /?q=user/login/

User-agent: Robozilla
    Crawl-delay: 10

    Allow: /site/robots.txt
    Allow: /site/index.php

    # Directories
    Disallow: /tracker/
    Disallow: /xtracker/
    Disallow: /user/
    Disallow: /book/export/
    Disallow: /forward/
    Disallow: /database/
    Disallow: /includes/
    Disallow: /misc/
    Disallow: /modules/
    Disallow: /sites/
    Disallow: /themes/
    Disallow: /scripts/
    Disallow: /updates/
    Disallow: /profiles/
    Disallow: /comment/
    Disallow: /taxonomy/
    Disallow: /popular/
    Disallow: */feed*
    Disallow: */comment/reply/
    Disallow: /popular/*
    Disallow: /comments*
    Disallow: /frontpage*
    Disallow: /comments*
    Disallow: /aggregator*
    Disallow: /aggregator2*
    Disallow: *?sort=asc&order=Time*
    Disallow: *comment-*
    # Files
    Disallow: /rss.xml
    Disallow: /feed
    Disallow: /xmlrpc.php
    Disallow: /cron.php
    Disallow: /update.php
    Disallow: /install.php
    Disallow: /INSTALL.mysql.txt
    Disallow: /INSTALL.pgsql.txt
    Disallow: /CHANGELOG.txt
    Disallow: /MAINTAINERS.txt
    Disallow: /LICENSE.txt
    Disallow: /UPGRADE.txt

    # Paths (clean URLs)
    Disallow: /admin/
    Disallow: /comment/reply/
    Disallow: /contact/
    Disallow: /logout/
    Disallow: /node/add/
    Disallow: /search/
    Disallow: /user/register/
    Disallow: /user/password/
    Disallow: /user/login/

    # Paths (no clean URLs)
    Disallow: /?q=admin/
    Disallow: /?q=comment/reply/
    Disallow: /?q=contact/
    Disallow: /?q=logout/
    Disallow: /?q=node/add/
    Disallow: /?q=search/
    Disallow: /?q=user/password/
    Disallow: /?q=user/register/
    Disallow: /?q=user/login/

User-agent: Nutch
    Crawl-delay: 10

    Allow: /site/robots.txt
    Allow: /site/index.php

    # Directories
    Disallow: /tracker/
    Disallow: /xtracker/
    Disallow: /user/
    Disallow: /book/export/
    Disallow: /forward/
    Disallow: /database/
    Disallow: /includes/
    Disallow: /misc/
    Disallow: /modules/
    Disallow: /sites/
    Disallow: /themes/
    Disallow: /scripts/
    Disallow: /updates/
    Disallow: /profiles/
    Disallow: /comment/
    Disallow: /taxonomy/
    Disallow: /popular/
    Disallow: */feed*
    Disallow: */comment/reply/
    Disallow: /popular/*
    Disallow: /comments*
    Disallow: /frontpage*
    Disallow: /comments*
    Disallow: /aggregator*
    Disallow: /aggregator2*
    Disallow: *?sort=asc&order=Time*
    Disallow: *comment-*
    # Files
    Disallow: /rss.xml
    Disallow: /feed
    Disallow: /xmlrpc.php
    Disallow: /cron.php
    Disallow: /update.php
    Disallow: /install.php
    Disallow: /INSTALL.mysql.txt
    Disallow: /INSTALL.pgsql.txt
    Disallow: /CHANGELOG.txt
    Disallow: /MAINTAINERS.txt
    Disallow: /LICENSE.txt
    Disallow: /UPGRADE.txt

    # Paths (clean URLs)
    Disallow: /admin/
    Disallow: /comment/reply/
    Disallow: /contact/
    Disallow: /logout/
    Disallow: /node/add/
    Disallow: /search/
    Disallow: /user/register/
    Disallow: /user/password/
    Disallow: /user/login/

    # Paths (no clean URLs)
    Disallow: /?q=admin/
    Disallow: /?q=comment/reply/
    Disallow: /?q=contact/
    Disallow: /?q=logout/
    Disallow: /?q=node/add/
    Disallow: /?q=search/
    Disallow: /?q=user/password/
    Disallow: /?q=user/register/
    Disallow: /?q=user/login/

User-agent: ia_archiver
    Crawl-delay: 10

    Allow: /site/robots.txt
    Allow: /site/index.php

    # Directories
    Disallow: /tracker/
    Disallow: /xtracker/
    Disallow: /user/
    Disallow: /book/export/
    Disallow: /forward/
    Disallow: /database/
    Disallow: /includes/
    Disallow: /misc/
    Disallow: /modules/
    Disallow: /sites/
    Disallow: /themes/
    Disallow: /scripts/
    Disallow: /updates/
    Disallow: /profiles/
    Disallow: /comment/
    Disallow: /taxonomy/
    Disallow: /popular/
    Disallow: */feed*
    Disallow: */comment/reply/
    Disallow: /popular/*
    Disallow: /comments*
    Disallow: /frontpage*
    Disallow: /comments*
    Disallow: /aggregator*
    Disallow: /aggregator2*
    Disallow: *?sort=asc&order=Time*
    Disallow: *comment-*
    # Files
    Disallow: /rss.xml
    Disallow: /feed
    Disallow: /xmlrpc.php
    Disallow: /cron.php
    Disallow: /update.php
    Disallow: /install.php
    Disallow: /INSTALL.mysql.txt
    Disallow: /INSTALL.pgsql.txt
    Disallow: /CHANGELOG.txt
    Disallow: /MAINTAINERS.txt
    Disallow: /LICENSE.txt
    Disallow: /UPGRADE.txt

    # Paths (clean URLs)
    Disallow: /admin/
    Disallow: /comment/reply/
    Disallow: /contact/
    Disallow: /logout/
    Disallow: /node/add/
    Disallow: /search/
    Disallow: /user/register/
    Disallow: /user/password/
    Disallow: /user/login/

    # Paths (no clean URLs)
    Disallow: /?q=admin/
    Disallow: /?q=comment/reply/
    Disallow: /?q=contact/
    Disallow: /?q=logout/
    Disallow: /?q=node/add/
    Disallow: /?q=search/
    Disallow: /?q=user/password/
    Disallow: /?q=user/register/
    Disallow: /?q=user/login/

User-agent: baiduspider
    Crawl-delay: 10

    Allow: /site/robots.txt
    Allow: /site/index.php

    # Directories
    Disallow: /tracker/
    Disallow: /xtracker/
    Disallow: /user/
    Disallow: /book/export/
    Disallow: /forward/
    Disallow: /database/
    Disallow: /includes/
    Disallow: /misc/
    Disallow: /modules/
    Disallow: /sites/
    Disallow: /themes/
    Disallow: /scripts/
    Disallow: /updates/
    Disallow: /profiles/
    Disallow: /comment/
    Disallow: /taxonomy/
    Disallow: /popular/
    Disallow: */feed*
    Disallow: */comment/reply/
    Disallow: /popular/*
    Disallow: /comments*
    Disallow: /frontpage*
    Disallow: /comments*
    Disallow: /aggregator*
    Disallow: /aggregator2*
    Disallow: *?sort=asc&order=Time*
    Disallow: *comment-*
    # Files
    Disallow: /rss.xml
    Disallow: /feed
    Disallow: /xmlrpc.php
    Disallow: /cron.php
    Disallow: /update.php
    Disallow: /install.php
    Disallow: /INSTALL.mysql.txt
    Disallow: /INSTALL.pgsql.txt
    Disallow: /CHANGELOG.txt
    Disallow: /MAINTAINERS.txt
    Disallow: /LICENSE.txt
    Disallow: /UPGRADE.txt

    # Paths (clean URLs)
    Disallow: /admin/
    Disallow: /comment/reply/
    Disallow: /contact/
    Disallow: /logout/
    Disallow: /node/add/
    Disallow: /search/
    Disallow: /user/register/
    Disallow: /user/password/
    Disallow: /user/login/

    # Paths (no clean URLs)
    Disallow: /?q=admin/
    Disallow: /?q=comment/reply/
    Disallow: /?q=contact/
    Disallow: /?q=logout/
    Disallow: /?q=node/add/
    Disallow: /?q=search/
    Disallow: /?q=user/password/
    Disallow: /?q=user/register/
    Disallow: /?q=user/login/