# Global rules
User-agent: *
# Allow general access to site assets needed for rendering
Allow: /assets/
Allow: /lib/

# Allow important public pages
Allow: /Home/
Allow: /Course/
Allow: /Search/
Allow: /Identity/Account/Login
Allow: /Identity/Account/Register
Allow: /Identity/Account/ForgotPassword
Allow: /Identity/Account/ResetPassword

# Disallow sensitive areas
Disallow: /Identity/Account/Manage/
Disallow: /Identity/Account/Logout
Disallow: /Admin/
Disallow: /Error/
Disallow: /api/
Disallow: /uploads/
Disallow: /templates/

# Allow sitemap and important files
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /favicon.ico

# Specific rules for different bots
# User-agent: Googlebot
# Allow: / # If you need to override a general disallow for Googlebot specifically

# User-agent: Bingbot
# Allow: / # If you need to override a general disallow for Bingbot specifically

User-agent: Slurp
Crawl-delay: 15

User-agent: DuckDuckBot
Crawl-delay: 15

# Block bad bots
User-agent: BadBot
Disallow: /

# Sitemap location
Sitemap: https://ujdigital.uj.ac.za/sitemap.xml