######
## This content is managed by CloudFlare worker monin-com-robots
######

#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# Remove Language Selection from Search Resules - applies to "www.monin.com/$"
# "$" means end of line
#Disallow: /$
# CSS, JS, Images
Allow: /misc/*.css$
Allow: /misc/*.css?
Allow: /misc/*.js$
Allow: /misc/*.js?
Allow: /misc/*.gif
Allow: /misc/*.jpg
Allow: /misc/*.jpeg
Allow: /misc/*.png
Allow: /modules/*.css$
Allow: /modules/*.css?
Allow: /modules/*.js$
Allow: /modules/*.js?
Allow: /modules/*.gif
Allow: /modules/*.jpg
Allow: /modules/*.jpeg
Allow: /modules/*.png
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /themes/*.css$
Allow: /themes/*.css?
Allow: /themes/*.js$
Allow: /themes/*.js?
Allow: /themes/*.gif
Allow: /themes/*.jpg
Allow: /themes/*.jpeg
Allow: /themes/*.png
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /us/hand-sanitizer*
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/
# Disallow: /maple-pumpkin-cold-brew-coffee-concentrate
# Disallow: /us/maple-pumpkin-cold-brew-coffee-concentrate
# Disallow: /ca/maple-pumpkin-cold-brew-coffee-concentrate
# Disallow: /spicy-agave
# Disallow: /us/spicy-agave
# Disallow: /ca/spicy-agave
Disallow: */black-friday-holiday

# hidden from web searches
Disallow: /us/authorized-seller-program
Disallow: /us/authorized-reseller-policy-us
Disallow: /us/authorized-distributor-policy-us
Disallow: /us/authorized-subdistributor-policy-us
Disallow: /us/publicwebsiteapproval
Disallow: /us/monin-satisfaction-guarantee
Disallow: /us/egg-hunt-purees
Disallow: /ca/egg-hunt-purees
Disallow: /us/egg-hunt-syrups
Disallow: /ca/egg-hunt-syrups
Disallow: /us/egg-hunt-sugar-free
Disallow: /ca/egg-hunt-sugar-free
Disallow: /us/egg-hunt-sauce
Disallow: /ca/egg-hunt-sauce
Disallow: /us/gift*

# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/


# Parameters to prevent server overload
Disallow: /*?customFilters
Disallow: /*?priceMin
Disallow: /*?priceMax
Disallow: /*?sortKey
Disallow: /*?sortDirection


# # Robots.txt for Community and Enterprise
 
# # GENERAL SETTINGS
 
# Enables robots.txt rules for all crawlers
 
User-agent: *
 
# # Crawl-delay parameter: the number of seconds you want to wait between successful requests to the same server.
# # Set a crawl rate, if your server's traffic problems. Please note that Google ignore crawl-delay setting in Robots.txt. You can set up this in Google Webmaster tool
# Crawl-delay: 30
 
# # Magento sitemap: URL to your sitemap file in Magento
# Sitemap: http://www.mystore.com/sitemap/sitemap.xml

Sitemap: https://www.monin.com/media/us_sitemap.xml
Sitemap: https://www.monin.com/media/spa_sitemap.xml
Sitemap: https://www.monin.com/media/ca_en_sitemap.xml
Sitemap: https://www.monin.com/media/ca_fr_sitemap.xml
 
# # Settings that relate to the UNDER CONSTRUCTION

#Disable some promotional forms
Disallow: /*/cf/*
Disallow: /*/nrn
Disallow: /*/nrn/*
 
# # Do not allow indexing files and folders that are required during development: CVS, SVN directory and dump files
Disallow: /*/*.cvs
Disallow: /*/*.svn
Disallow: /*/*.idea
Disallow: /*/*.sql
Disallow: /*/*.tgz
 
# # GENERAL SETTINGS For MAGENTO
 
# # Do not index the general technical Magento directory
Disallow: /*/app/
Allow: /*/app/help
Allow: /*/app/help/*
 
# # Do not index the shared files
Disallow: /*/*api.php
Disallow: /*/*cron.php
Disallow: /*/*cron.sh
Disallow: /*/*error_log
Disallow: /*/*get.php
Disallow: /*/*install.php
Disallow: /*/*LICENSE.html
Disallow: /*/*LICENSE.txt
Disallow: /*/*LICENSE_AFL.txt
Disallow: /*/*README.txt
Disallow: /*/*RELEASE_NOTES.txt
 
# # Do not index the page subcategories that are sorted or filtered.
Disallow: /*/*?dir* 
Disallow: /*/*?limit*
Disallow: /*/*?mode*
 
# # Do not index the second copy of the home page (example.com / index.php /). Un-comment only if you have activated Magento SEO URLs.
# # Disallow: /*/index.php/
 
# # Do not index the link from the session ID
Disallow: /*/*?*sid=
Disallow: /*/*?*Sid=
Disallow: /*/*?*SId=
Disallow: /*/*?*sid=
Disallow: /*SID=”
 
# # Do not index the page checkout and user account
Disallow: /*/checkout/
Disallow: /*/onestepcheckout
Disallow: /*/customer/
Disallow: /*/customer/account
Disallow: /*/customer/account/login
Disallow: /*/catalogsearch
Disallow: */search/
Disallow: */my-account
 
# # Do not index the search page and CEO, non-optimized link categories
Disallow: /*/catalogsearch
Disallow: /*/catalog/product_compare
Disallow: /*/catalog/category/view
Disallow: /*/catalog/product/view
Disallow: */review/
Disallow: */sendfriend/



# # Do not index the Logo page
Disallow: /us/logo

# # Server Settings
 
# # Do not index the general technical directories and files on a server
Disallow: /*/cgi-bin
Disallow: /*/cleanup.php
Disallow: /*/apc.php
Disallow: /*/memcache.php
Disallow: /*/phpinfo.php
 

###
# French Website and EMEA website
Disallow: /fr/lib/
Disallow: /fr/*.php$
Disallow: /fr/pkginfo/
Disallow: /fr/report/
Disallow: /fr/var/
Disallow: /fr/customer/
Disallow: /fr/sendfriend/
Disallow: /fr/review/
Disallow: /fr/*SID=
Disallow: */media/captcha 
Disallow: /emea/monin-intranet

###Disallow BR IN and APAC until live released  
Disallow: /br/*
Disallow: /apac/*


###
# SEMRush handles allow/disallow differently than standard bots.
# Ensure here that SEMRush can access subdirectories of www.monin.com
User-agent: SemrushBot-SA
Allow: /