#Block Certain Bots:
User-Agent: TweetMeMeBot
Disallow: /

#Block Certain Bots:
User-Agent: FatBot
Disallow: /

#Block Certain Bots:
User-Agent: AhrefsBot
Disallow: /

#Block Certain Bots:
User-agent: PetalBot
Disallow: /

#Block Certain Bots:
User-agent: Applebot
Disallow: /

User-agent: Googlebot
Disallow: /css
Disallow: /include
Disallow: /akamai
Disallow: /LBKA
Disallow: /jsp/compliance
Disallow: /jsp/offer/recr/uk/common/landing.jsp
Disallow: /jsp/registration
Disallow: /jsp/account
Disallow: /jsp/checkout
Disallow: /DWBase
Disallow: /cart
#Disallow: /jsp/curity
Disallow: /jsp/templates/error_templates/404.jsp
Disallow: /search
Allow: /wines

User-agent: Bingbot
Disallow: /css
Disallow: /include
Disallow: /akamai
Disallow: /LBKA
Disallow: /jsp/compliance
Disallow: /jsp/offer/recr/uk/common/landing.jsp
Disallow: /jsp/registration
Disallow: /jsp/account
Disallow: /jsp/checkout
Disallow: /DWBase
Disallow: /cart
#Disallow: /jsp/curity
Disallow: /jsp/templates/error_templates/404.jsp
Disallow: /search
Allow: /wines

User-agent: *
Crawl-delay: 5

#Allow All Others But Block Certain Pages:
User-agent: *
Disallow: /css
Disallow: /include
Disallow: /akamai
Disallow: /LBKA
Disallow: /jsp/compliance
#Disallow: /jsp/offer
Disallow:/jsp/offer/recr/uk/common/landing.jsp
Disallow: /jsp/registration
Disallow: /jsp/account
Disallow: /jsp/checkout
Disallow: /DWBase
Disallow: /cart
Disallow: /jsp/curity
#Disallow: /jsp/offer/cm/common/offer.jsp
Disallow: /jsp/templates/error_templates/404.jsp
Disallow: /wines
Disallow: /search

#sitemap links
Sitemap: https://www.wsjwine.com/sitemap.xml