User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: *
Disallow: /*-or-*
Disallow:
Crawl-delay: 5   # 5 seconden tussen requests (niet door Google gebruikt)


# Block Facebook/Meta crawler on faceted search
User-agent: meta-externalagent
Disallow: /vacatures/vakgebied-
Disallow: /vacatures/regio-
Disallow: /vacatures/sector-
Disallow: /vacatures-test/
Crawl-delay: 10

User-agent: facebookexternalhit
Disallow: /vacatures/vakgebied-
Disallow: /vacatures/regio-
Disallow: /vacatures/sector-
Disallow: /vacatures-test/
Crawl-delay: 10

# robots.txt - bots blokkeren
User-agent: Bytespider Disallow: 
User-agent: MJ12bot Disallow: 
User-agent: PetalBot Disallow:
User-agent: YandexBot Disallow:
# Bingbot beperken (crawl-delay) 
User-agent: bingbot Crawl-delay: 10