# Default modx exclusions
User-agent: *
Disallow: /members/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /manifests/
Disallow: /users/
Disallow: /profile/
Disallow: /crm/
Disallow: /assets/export/
Disallow: /assets/import/
Disallow: /assets/modules/
Disallow: /assets/plugins/
Disallow: /assets/snippets/
Disallow: /assets/files/
Disallow: /install/
Disallow: /admin/
Disallow: /cms/
Disallow: /manager/
Disallow: /partners/
Disallow: /bzz/
Disallow: /redactie/
Disallow: /packages/
Disallow: /video/
Disallow: /auth/
Disallow: /account/
Disallow: /preview/
Disallow: /search/
Disallow: /cgi-bin/

# For sitemaps.xml autodiscovery. Uncomment if you have one:
Sitemap: https://jasperjaspers.nl/sitemap.xml
Sitemap: https://jasperjaspers.nl/xml-sitemap.php

# De AhrefsBot mag maar 1 keer per dag langskomen
User-agent: AhrefsBot 
Crawl-Delay: 86400

# The example for img2dataset, although the default is *None*
# User-agent: img2dataset
# Disallow: /

# Brandwatch - "AI to discover new trends"
# User-agent: magpie-crawler
# Disallow: /

# webz.io - they sell data for training LLMs.
# User-agent: Omgilibot
# Disallow: /

# Items below were sourced from darkvisitors.com
# Categories included: "AI Data Scraper", "AI Assistant", "AI Search Crawler", "Undocumented AI Agent"

# AI Search Crawler
# https://darkvisitors.com/agents/amazonbot

# User-agent: Amazonbot
# Disallow: /

# Undocumented AI Agent
# https://darkvisitors.com/agents/anthropic-ai

# User-agent: anthropic-ai
# Disallow: /

# AI Search Crawler
# https://darkvisitors.com/agents/applebot

# User-agent: Applebot
# Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/applebot-extended

# User-agent: Applebot-Extended
# Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/bytespider

# User-agent: Bytespider
# Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/ccbot

# User-agent: CCBot
# Disallow: /

# AI Assistant
# https://darkvisitors.com/agents/chatgpt-user

# User-agent: ChatGPT-User
# Disallow: /

# Undocumented AI Agent
# https://darkvisitors.com/agents/claude-web

# User-agent: Claude-Web
# Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/claudebot

# User-agent: ClaudeBot
# Disallow: /

# Undocumented AI Agent
# https://darkvisitors.com/agents/cohere-ai

# User-agent: cohere-ai
# Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/diffbot

# User-agent: Diffbot
# Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/facebookbot

# User-agent: FacebookBot
# Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/google-extended

# User-agent: Google-Extended
# Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/gptbot

# User-agent: GPTBot
# Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/omgili

# User-agent: omgili
# Disallow: /

# AI Search Crawler
# https://darkvisitors.com/agents/perplexitybot

# User-agent: PerplexityBot
# Disallow: /

# AI Search Crawler
# https://darkvisitors.com/agents/youbot

# User-agent: YouBot
# Disallow: /