# general
User-agent: *
Disallow: /distfiles/
User-agent: *
Disallow: /portdb/

# semrushbot
User-agent: SemrushBot
Disallow: /
User-agent: SiteAuditBot
Disallow: /
User-agent: SemrushBot-SI
Disallow: /
User-agent: SemrushBot-SWA
Disallow: /
User-agent: SplitSignalBot
Disallow: /
User-agent: SemrushBot-OCOB
Disallow: /
User-agent: Semrush*
Disallow: /

# petalbot
User-agent: PetalBot
Disallow: /

# facebook/meta bot
User-agent: meta-externalagent
Disallow: /
User-agent: meta*
Disallow: /
User-agent: facebookexternalhit
Disallow: /
User-agent: facebookexternalhit*
Disallow: /

# mj12bot?
User-agent: MJ12bot
Disallow: /
User-agent: MJ12bot*
Disallow: /

# bytespider
User-agent: Bytespider
Disallow: /
User-agent: Bytespider*
Disallow: /

# dotbot
User-agent: DotBot
Disallow: /
User-agent: DotBot*
Disallow: /

# BotPoke?
User-agent: BotPoke
Disallow: /
User-agent: BotPoke*
Disallow: /

# CensysInspect
User-agent: CensysInspect
Disallow: /
User-agent: CensysInspect*
Disallow: /

# bingbot
User-agent: bingbot
Disallow: /
User-agent: bingbot*
Disallow: /

# yandex
User-agent: YandexBot
Disallow: /
User-agent: YandexBot*
Disallow: /

# InternetMeasurement
User-agent: InternetMeasurement
Disallow: /
User-agent: InternetMeasurement*
Disallow: /