#
# Hello bot! There are some routes that need authentication. You should avoid them :)
#

User-agent: MJ12bot
Disallow: /profile-settings
Disallow: /l/new
Disallow: /l/*/checkout
Disallow: /l/*/draft
Disallow: /l/*/pending-approval
Disallow: /l/*/new
Disallow: /l/*/edit
Disallow: /inbox
Disallow: /order
Disallow: /sale
Disallow: /listings
Disallow: /account
Disallow: /reset-password
Disallow: /verify-email
Disallow: /preview
Disallow: /styleguide
Crawl-Delay: 20

User-agent: AhrefsBot
Disallow: /profile-settings
Disallow: /l/new
Disallow: /l/*/checkout
Disallow: /l/*/draft
Disallow: /l/*/pending-approval
Disallow: /l/*/new
Disallow: /l/*/edit
Disallow: /inbox
Disallow: /order
Disallow: /sale
Disallow: /listings
Disallow: /account
Disallow: /reset-password
Disallow: /verify-email
Disallow: /preview
Disallow: /styleguide
Crawl-Delay: 20

User-agent: SemrushBot
Disallow: /profile-settings
Disallow: /l/new
Disallow: /l/*/checkout
Disallow: /l/*/draft
Disallow: /l/*/pending-approval
Disallow: /l/*/new
Disallow: /l/*/edit
Disallow: /inbox
Disallow: /order
Disallow: /sale
Disallow: /listings
Disallow: /account
Disallow: /reset-password
Disallow: /verify-email
Disallow: /preview
Disallow: /styleguide
Crawl-Delay: 20

# AI training bots
User-agent: CCBot
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Meta-ExternalAgent
User-agent: Applebot-Extended
Disallow: /

User-agent: *
Disallow: /profile-settings
Disallow: /l/new
Disallow: /l/*/checkout
Disallow: /l/*/draft
Disallow: /l/*/pending-approval
Disallow: /l/*/new
Disallow: /l/*/edit
Disallow: /inbox
Disallow: /order
Disallow: /sale
Disallow: /listings
Disallow: /account
Disallow: /reset-password
Disallow: /verify-email
Disallow: /preview
Disallow: /styleguide
Crawl-Delay: 5

Sitemap: https://www.thetoylibrary.ie/sitemap-index.xml