## Robots.txt ## GENERAL SETTINGS #Enables robots.txt rules for all crawlers User-agent: * ## Crawl-delay parameter: the number of seconds you want to wait between successful requests to the same server. ## Set a crawl rate, if your server's traffic problems. Please note that Google ignore crawl-delay setting in Robots.txt. You can set up this in Google Webmaster tool Crawl-delay: 90 ## sitemap: URL to your sitemap file Sitemap: https://www.gempundit.com/sitemap_index.xml ## Settings that relate to the UNDER CONSTRUCTION ## Do not allow indexing files and folders that are required during development: CVS, SVN directory and dump files Disallow: /CVS Disallow: /*.Svn$ Disallow: /*.Idea$ Disallow: /*.Sql$ Disallow: /*.Tgz$ ## GENERAL SETTINGS ## Do not index the page Disallow: /admin/ ## Do not index the general technical directory Disallow: /404/ Disallow: /app/ Disallow: /downloader/ Disallow: /errors/ Disallow: /includes/ Disallow: /js/ Disallow: /lib/ Disallow: /magento/ Disallow: /media/ Disallow: /pkginfo/ Disallow: /report/ Disallow: /scripts/ Disallow: /shell/ Disallow: /skin/ Disallow: /stats/ Disallow: /var/ ## Do not index the shared files Disallow: /api.php Disallow: /cron.php Disallow: /cron.sh Disallow: /error_log Disallow: /get.php Disallow: /install.php Disallow: /LICENSE.html Disallow: /LICENSE.txt Disallow: /LICENSE_AFL.txt Disallow: /README.txt Disallow: /RELEASE_NOTES.txt ## Do not index the page subcategories that are sorted or filtered. Disallow: /*?Dir* Disallow: /*?Dir=desc Disallow: /*?Dir=asc Disallow: /*?Limit=all Disallow: /*?Mode* ## Do not index the second copy of the home page (example.com/ index.php/). Un-comment only if you have activated SEO URLs. Disallow: /index.php/ ## Do not index the link from the session ID Disallow: /*?SID= ## Do not index the page checkout and user account Disallow: /checkout/ Disallow: /onestepcheckout/ Disallow: /customer/ Disallow: /customer/account/ Disallow: /customer/account/login/ ## Do not index the search page and CEO, non-optimized link categories Disallow: /catalogsearch/ Disallow: /catalog/product_compare/ Disallow: /catalog/category/view/ Disallow: /catalog/product/view/ Disallow: /directory/currency/switch/currency/ Disallow: /directory Disallow: /catalog/seo_sitemap/ Disallow: /miscellaneous/index/ Disallow: /search/ Disallow: /offers/ #Do not index the directory Disallow: /directory/currency/switch/currency/ ## Server Settings ## Do not index the general technical directories and files on a server Disallow: /cgi-bin/ Disallow: /cleanup.php Disallow: /apc.php Disallow: /memcache.php Disallow: /phpinfo.php User-agent: AhrefsBot Crawl-delay: 10 Disallow: /admin/ Disallow: /CVS Disallow: /*.Svn$ Disallow: /*.Idea$ Disallow: /*.Sql$ Disallow: /*.Tgz$ Disallow: /app/ Disallow: /downloader/ Disallow: /errors/ Disallow: /includes/ Disallow: /js/ Disallow: /lib/ Disallow: /magento/ Disallow: /pkginfo/ Disallow: /report/ Disallow: /scripts/ Disallow: /shell/ Disallow: /skin/ Disallow: /stats/ Disallow: /var/ Disallow: /api.php Disallow: /cron.php Disallow: /cron.sh Disallow: /error_log Disallow: /get.php Disallow: /install.php Disallow: /LICENSE.html Disallow: /LICENSE.txt Disallow: /LICENSE_AFL.txt Disallow: /README.txt Disallow: /RELEASE_NOTES.txt Disallow: /*?Dir* Disallow: /*?Dir=desc Disallow: /*?Dir=asc Disallow: /*?Limit=all Disallow: /*?Mode* Disallow: /*?SID= Disallow: /index.php/ Disallow: /checkout/ Disallow: /onestepcheckout/ Disallow: /customer/ Disallow: /customer/account/ Disallow: /customer/account/login/ Disallow: /catalogsearch/ Disallow: /catalog/product_compare/ Disallow: /catalog/category/view/ Disallow: /catalog/product/view/ Disallow: /directory/currency/switch/currency/ Disallow: /directory Disallow: /catalog/seo_sitemap/ Disallow: /miscellaneous/index/ Disallow: /cgi-bin/ Disallow: /cleanup.php Disallow: /apc.php Disallow: /memcache.php Disallow: /phpinfo.php Disallow: /offers/ Disallow: /search Disallow: /apple-app-site-association User-agent: AhrefsSiteAudit Crawl-delay: 10 Disallow: /admin/ Disallow: /CVS Disallow: /*.Svn$ Disallow: /*.Idea$ Disallow: /*.Sql$ Disallow: /*.Tgz$ Disallow: /app/ Disallow: /downloader/ Disallow: /errors/ Disallow: /includes/ Disallow: /js/ Disallow: /lib/ Disallow: /magento/ Disallow: /pkginfo/ Disallow: /report/ Disallow: /scripts/ Disallow: /shell/ Disallow: /skin/ Disallow: /stats/ Disallow: /var/ Disallow: /api.php Disallow: /cron.php Disallow: /cron.sh Disallow: /error_log Disallow: /get.php Disallow: /install.php Disallow: /LICENSE.html Disallow: /LICENSE.txt Disallow: /LICENSE_AFL.txt Disallow: /README.txt Disallow: /RELEASE_NOTES.txt Disallow: /*?Dir* Disallow: /*?Dir=desc Disallow: /*?Dir=asc Disallow: /*?Limit=all Disallow: /*?Mode* Disallow: /*?SID= Disallow: /index.php/ Disallow: /checkout/ Disallow: /onestepcheckout/ Disallow: /customer/ Disallow: /customer/account/ Disallow: /customer/account/login/ Disallow: /catalogsearch/ Disallow: /catalog/product_compare/ Disallow: /catalog/category/view/ Disallow: /catalog/product/view/ Disallow: /directory/currency/switch/currency/ Disallow: /directory Disallow: /catalog/seo_sitemap/ Disallow: /miscellaneous/index/ Disallow: /cgi-bin/ Disallow: /cleanup.php Disallow: /apc.php Disallow: /memcache.php Disallow: /phpinfo.php Disallow: /offers/ Disallow: /search Disallow: /apple-app-site-association # Google adsbot ignores robots.txt unless specifically named! ## Google ads bots User-agent: Googlebot Disallow: /CVS Disallow: /*.Svn$ Disallow: /*.Idea$ Disallow: /*.Sql$ Disallow: /*.Tgz$ Disallow: /admin/ Disallow: /404/ Disallow: /app/ Disallow: /downloader/ Disallow: /errors/ Disallow: /includes/ Disallow: /js/ Disallow: /lib/ Disallow: /magento/ Disallow: /pkginfo/ Disallow: /report/ Disallow: /scripts/ Disallow: /shell/ Disallow: /skin/ Disallow: /stats/ Disallow: /var/ Disallow: /api.php Disallow: /cron.php Disallow: /cron.sh Disallow: /error_log Disallow: /get.php Disallow: /install.php Disallow: /LICENSE.html Disallow: /LICENSE.txt Disallow: /LICENSE_AFL.txt Disallow: /README.txt Disallow: /RELEASE_NOTES.txt Disallow: /*?Dir* Disallow: /*?Dir=desc Disallow: /*?Dir=asc Disallow: /*?Limit=all Disallow: /*?Mode* Disallow: /index.php/ Disallow: /*?SID= Disallow: /checkout/ Disallow: /onestepcheckout/ Disallow: /customer/ Disallow: /customer/account/ Disallow: /customer/account/login/ Disallow: /catalogsearch/ Disallow: /catalog/product_compare/ Disallow: /catalog/category/view/ Disallow: /catalog/product/view/ Disallow: /directory/currency/switch/currency/ Disallow: /directory Disallow: /catalog/seo_sitemap/ Disallow: /miscellaneous/index/ Disallow: /search/ Disallow: /offers/ Disallow: /cgi-bin/ Disallow: /cleanup.php Disallow: /apc.php Disallow: /memcache.php Disallow: /phpinfo.php Disallow: /search Disallow: /apple-app-site-association Disallow: /directory/ Disallow: /catalog/ Disallow: /miscellaneous/ Disallow: /facebookfree/ User-agent: AdsBot-Google Disallow: /CVS Disallow: /*.Svn$ Disallow: /*.Idea$ Disallow: /*.Sql$ Disallow: /*.Tgz$ Disallow: /admin/ Disallow: /404/ Disallow: /app/ Disallow: /downloader/ Disallow: /errors/ Disallow: /includes/ Disallow: /js/ Disallow: /lib/ Disallow: /magento/ Disallow: /pkginfo/ Disallow: /report/ Disallow: /scripts/ Disallow: /shell/ Disallow: /skin/ Disallow: /stats/ Disallow: /var/ Disallow: /api.php Disallow: /cron.php Disallow: /cron.sh Disallow: /error_log Disallow: /get.php Disallow: /install.php Disallow: /LICENSE.html Disallow: /LICENSE.txt Disallow: /LICENSE_AFL.txt Disallow: /README.txt Disallow: /RELEASE_NOTES.txt Disallow: /*?Dir* Disallow: /*?Dir=desc Disallow: /*?Dir=asc Disallow: /*?Limit=all Disallow: /*?Mode* Disallow: /index.php/ Disallow: /*?SID= Disallow: /checkout/ Disallow: /onestepcheckout/ Disallow: /customer/ Disallow: /customer/account/ Disallow: /customer/account/login/ Disallow: /catalogsearch/ Disallow: /catalog/product_compare/ Disallow: /catalog/category/view/ Disallow: /catalog/product/view/ Disallow: /directory/currency/switch/currency/ Disallow: /directory Disallow: /catalog/seo_sitemap/ Disallow: /miscellaneous/index/ Disallow: /search/ Disallow: /offers/ Disallow: /cgi-bin/ Disallow: /cleanup.php Disallow: /apc.php Disallow: /memcache.php Disallow: /phpinfo.php Disallow: /search Disallow: /apple-app-site-association Disallow: /directory/ Disallow: /catalog/ Disallow: /miscellaneous/ Disallow: /facebookfree/ User-agent: msnbot Disallow: /CVS Disallow: /*.Svn$ Disallow: /*.Idea$ Disallow: /*.Sql$ Disallow: /*.Tgz$ Disallow: /admin/ Disallow: /404/ Disallow: /app/ Disallow: /downloader/ Disallow: /errors/ Disallow: /includes/ Disallow: /js/ Disallow: /lib/ Disallow: /magento/ Disallow: /pkginfo/ Disallow: /report/ Disallow: /scripts/ Disallow: /shell/ Disallow: /skin/ Disallow: /stats/ Disallow: /var/ Disallow: /api.php Disallow: /cron.php Disallow: /cron.sh Disallow: /error_log Disallow: /get.php Disallow: /install.php Disallow: /LICENSE.html Disallow: /LICENSE.txt Disallow: /LICENSE_AFL.txt Disallow: /README.txt Disallow: /RELEASE_NOTES.txt Disallow: /*?Dir* Disallow: /*?Dir=desc Disallow: /*?Dir=asc Disallow: /*?Limit=all Disallow: /*?Mode* Disallow: /index.php/ Disallow: /*?SID= Disallow: /checkout/ Disallow: /onestepcheckout/ Disallow: /customer/ Disallow: /customer/account/ Disallow: /customer/account/login/ Disallow: /catalogsearch/ Disallow: /catalog/product_compare/ Disallow: /catalog/category/view/ Disallow: /catalog/product/view/ Disallow: /directory/currency/switch/currency/ Disallow: /directory Disallow: /catalog/seo_sitemap/ Disallow: /miscellaneous/index/ Disallow: /search/ Disallow: /offers/ Disallow: /cgi-bin/ Disallow: /cleanup.php Disallow: /apc.php Disallow: /memcache.php Disallow: /phpinfo.php Disallow: /search Disallow: /apple-app-site-association Disallow: /directory/ Disallow: /catalog/ Disallow: /miscellaneous/ Disallow: /facebookfree/ ## SETTINGS Image indexing ## Optional: If you do not want to Google and Bing to index your images # User-agent: Googlebot-Image # allow: / # User-agent: msnbot-media # allow: /