# robots.txt for http://anklang.testbit.eu/

Sitemap: https://anklang.testbit.eu/sitemap.xml

# Not interesting
User-agent: *
Disallow: /impressum

# Slow down hungry crawlers
User-agent: *
crawl-delay: 600

# allow genuine search engines
User-agent: Googlebot
Allow: /
User-agent: yandex
Allow: /
User-agent: bingbot
Allow: /

# Allow the Internet Archiver to index action=raw and thereby store the raw wikitext of pages
User-agent: ia_archiver
Allow: /*&action=raw

User-agent: *
Disallow: /wiki.mw/

# Block SEO bots (AhrefsBot visits are excessive)
User-agent: MJ12bot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: rogerbot
Disallow:/
User-agent: dotbot
Disallow:/
User-agent: AhrefsBot
Disallow: /
User-agent: Alexibot
Disallow: /
User-agent: SurveyBot
Disallow: /
User-agent: Xenu’s
Disallow: /
User-agent: Xenu’s Link Sleuth 1.1c
Disallow: /
User-agent: XoviBot
Disallow: /