#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
Crawl-delay: 10
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
Disallow: /group/includes/
Disallow: /group/misc/
Disallow: /group/modules/
Disallow: /group/profiles/
Disallow: /group/scripts/
Disallow: /group/themes/
Disallow: /bricks/includes/
Disallow: /bricks/misc/
Disallow: /bricks/modules/
Disallow: /bricks/profiles/
Disallow: /bricks/scripts/
Disallow: /bricks/themes/
Disallow: /ebrick-eboard/includes/
Disallow: /ebrick-eboard/misc/
Disallow: /ebrick-eboard/modules/
Disallow: /ebrick-eboard/profiles/
Disallow: /ebrick-eboard/scripts/
Disallow: /ebrick-eboard/themes/
Disallow: /huwa/includes/
Disallow: /huwa/misc/
Disallow: /huwa/modules/
Disallow: /huwa/profiles/
Disallow: /huwa/scripts/
Disallow: /huwa/themes/
Disallow: /professionals/includes/
Disallow: /professionals/misc/
Disallow: /professionals/modules/
Disallow: /professionals/profiles/
Disallow: /professionals/scripts/
Disallow: /professionals/themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
Disallow: /group/CHANGELOG.txt
Disallow: /group/cron.php
Disallow: /group/INSTALL.mysql.txt
Disallow: /group/INSTALL.pgsql.txt
Disallow: /group/INSTALL.sqlite.txt
Disallow: /group/install.php
Disallow: /group/INSTALL.txt
Disallow: /group/LICENSE.txt
Disallow: /group/MAINTAINERS.txt
Disallow: /group/update.php
Disallow: /group/UPGRADE.txt
Disallow: /group/xmlrpc.php
Disallow: /bricks/CHANGELOG.txt
Disallow: /bricks/cron.php
Disallow: /bricks/INSTALL.mysql.txt
Disallow: /bricks/INSTALL.pgsql.txt
Disallow: /bricks/INSTALL.sqlite.txt
Disallow: /bricks/install.php
Disallow: /bricks/INSTALL.txt
Disallow: /bricks/LICENSE.txt
Disallow: /bricks/MAINTAINERS.txt
Disallow: /bricks/update.php
Disallow: /bricks/UPGRADE.txt
Disallow: /bricks/xmlrpc.php
Disallow: /ebrick-eboard/CHANGELOG.txt
Disallow: /ebrick-eboard/cron.php
Disallow: /ebrick-eboard/INSTALL.mysql.txt
Disallow: /ebrick-eboard/INSTALL.pgsql.txt
Disallow: /ebrick-eboard/INSTALL.sqlite.txt
Disallow: /ebrick-eboard/install.php
Disallow: /ebrick-eboard/INSTALL.txt
Disallow: /ebrick-eboard/LICENSE.txt
Disallow: /ebrick-eboard/MAINTAINERS.txt
Disallow: /ebrick-eboard/update.php
Disallow: /ebrick-eboard/UPGRADE.txt
Disallow: /ebrick-eboard/xmlrpc.php
Disallow: /huwa/CHANGELOG.txt
Disallow: /huwa/cron.php
Disallow: /huwa/INSTALL.mysql.txt
Disallow: /huwa/INSTALL.pgsql.txt
Disallow: /huwa/INSTALL.sqlite.txt
Disallow: /huwa/install.php
Disallow: /huwa/INSTALL.txt
Disallow: /huwa/LICENSE.txt
Disallow: /huwa/MAINTAINERS.txt
Disallow: /huwa/update.php
Disallow: /huwa/UPGRADE.txt
Disallow: /huwa/xmlrpc.php
Disallow: /professionals/CHANGELOG.txt
Disallow: /professionals/cron.php
Disallow: /professionals/INSTALL.mysql.txt
Disallow: /professionals/INSTALL.pgsql.txt
Disallow: /professionals/INSTALL.sqlite.txt
Disallow: /professionals/install.php
Disallow: /professionals/INSTALL.txt
Disallow: /professionals/LICENSE.txt
Disallow: /professionals/MAINTAINERS.txt
Disallow: /professionals/update.php
Disallow: /professionals/UPGRADE.txt
Disallow: /professionals/xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /group/admin/
Disallow: /group/comment/reply/
Disallow: /group/filter/tips/
Disallow: /group/node/add/
Disallow: /group/search/
Disallow: /group/user/register/
Disallow: /group/user/password/
Disallow: /group/user/login/
Disallow: /group/user/logout/
Disallow: /bricks/admin/
Disallow: /bricks/comment/reply/
Disallow: /bricks/filter/tips/
Disallow: /bricks/node/add/
Disallow: /bricks/search/
Disallow: /bricks/user/register/
Disallow: /bricks/user/password/
Disallow: /bricks/user/login/
Disallow: /bricks/user/logout/
Disallow: /ebrick-eboard/admin/
Disallow: /ebrick-eboard/comment/reply/
Disallow: /ebrick-eboard/filter/tips/
Disallow: /ebrick-eboard/node/add/
Disallow: /ebrick-eboard/search/
Disallow: /ebrick-eboard/user/register/
Disallow: /ebrick-eboard/user/password/
Disallow: /ebrick-eboard/user/login/
Disallow: /ebrick-eboard/user/logout/
Disallow: /huwa/admin/
Disallow: /huwa/comment/reply/
Disallow: /huwa/filter/tips/
Disallow: /huwa/node/add/
Disallow: /huwa/search/
Disallow: /huwa/user/register/
Disallow: /huwa/user/password/
Disallow: /huwa/user/login/
Disallow: /huwa/user/logout/
Disallow: /professionals/admin/
Disallow: /professionals/comment/reply/
Disallow: /professionals/filter/tips/
Disallow: /professionals/node/add/
Disallow: /professionals/search/
Disallow: /professionals/user/register/
Disallow: /professionals/user/password/
Disallow: /professionals/user/login/
Disallow: /professionals/user/logout/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
Disallow: /group/?q=admin/
Disallow: /group/?q=comment/reply/
Disallow: /group/?q=filter/tips/
Disallow: /group/?q=node/add/
Disallow: /group/?q=search/
Disallow: /group/?q=user/password/
Disallow: /group/?q=user/register/
Disallow: /group/?q=user/login/
Disallow: /group/?q=user/logout/
Disallow: /bricks/?q=admin/
Disallow: /bricks/?q=comment/reply/
Disallow: /bricks/?q=filter/tips/
Disallow: /bricks/?q=node/add/
Disallow: /bricks/?q=search/
Disallow: /bricks/?q=user/password/
Disallow: /bricks/?q=user/register/
Disallow: /bricks/?q=user/login/
Disallow: /bricks/?q=user/logout/
Disallow: /ebrick-eboard/?q=admin/
Disallow: /ebrick-eboard/?q=comment/reply/
Disallow: /ebrick-eboard/?q=filter/tips/
Disallow: /ebrick-eboard/?q=node/add/
Disallow: /ebrick-eboard/?q=search/
Disallow: /ebrick-eboard/?q=user/password/
Disallow: /ebrick-eboard/?q=user/register/
Disallow: /ebrick-eboard/?q=user/login/
Disallow: /ebrick-eboard/?q=user/logout/
Disallow: /huwa/?q=admin/
Disallow: /huwa/?q=comment/reply/
Disallow: /huwa/?q=filter/tips/
Disallow: /huwa/?q=node/add/
Disallow: /huwa/?q=search/
Disallow: /huwa/?q=user/password/
Disallow: /huwa/?q=user/register/
Disallow: /huwa/?q=user/login/
Disallow: /huwa/?q=user/logout/
Disallow: /professionals/?q=admin/
Disallow: /professionals/?q=comment/reply/
Disallow: /professionals/?q=filter/tips/
Disallow: /professionals/?q=node/add/
Disallow: /professionals/?q=search/
Disallow: /professionals/?q=user/password/
Disallow: /professionals/?q=user/register/
Disallow: /professionals/?q=user/login/
Disallow: /professionals/?q=user/logout/

# Found this by awk -F\" '{print $6}' access.log | sort | uniq -c | sort -fr

User-agent: BLEXBot
Disallow: /
User-Agent: SputnikBot
Disallow: /
User-agent: SemrushBot
Disallow: /

User-Agent: Googlebot
Allow: .js
Allow: .css