User-agent: *
Disallow: /cgi-bin/
Disallow: /members/
Disallow: /member.php
Disallow: /newreply.php
Disallow: /search.php
Disallow: /install/
Disallow: /profile.php
Disallow: /register.php
Disallow: /report.php
Crawl-Delay: 4
Sitemap: http://goteen.net/xmlsitemap.php


# Our policy
#
# Allowed:
# - Search engine indexers
# - Archival services (e.g. IA)
#
# Disallowed:
# - Marketing or SEO crawlers
# - Bots which are too agressive by default. This is subjective, if you annoy
#   our sysadmins you'll be blocked.
#
# https://github.com/NC3-LU/MOSP/blob/140d30d13c996689e22770a4f6811bc26f3f8f0c/mosp/templates/robots.txt
# Reach out to opensource@nc3.lu if you have questions.

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: AspiegelBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-Agent: NPBot
Disallow: /

User-Agent: SlySearch
Disallow: /