# www.kollegeninfo.at #

# For all robots
User-agent: *
Allow: /

Disallow: */?
Disallow: */account/
Disallow: */checkout/
Disallow: */widgets/
Disallow: */navigation/
Disallow: */bundles/

Disallow: */impressum$
Disallow: */datenschutz$
Disallow: */agb$

Sitemap: https://www.kollegeninfo.at/sitemap.xml

# Allow CSS and JS, according to Google’s Gary Illyes:
User-agent: Googlebot
Allow: .js
Allow: .css

# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /

# Block MJ12bot as it is just noise
User-agent: MJ12bot
Disallow: /

# Block dotbot as it cannot parse base urls properly
User-agent: dotbot/1.0
Disallow: /

# Block Gigabot
User-agent: Gigabot
Disallow: /