Disallow: /cgi-bin/

# Teleport Pro is too dumb, rude and agressive
User-agent: Teleport Pro
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: CCBot
Crawl-delay: 120

User-agent: msnbot
Crawl-delay: 120

User-agent: Slurp
Crawl-delay: 120

User-agent: ScoutJet
Crawl-delay: 60

User-agent: Blekkobot
Crawl-delay: 60

User-agent: Yandex
Crawl-delay: 120

User-agent: HyperRobot
Disallow: /

User-agent: 008
Disallow: /

User-agent: Teoma
Crawl-delay: 120

# Voila bot is too agressive, and there is no real benefit in being listed in voila.com
User-agent: VoilaBot
Disallow: /

# iltrovatore bot is too agressive, and there is no real benefit in being listed in iltrovatore
User-agent: iltrovatore-Setaccio
Disallow: /

User-agent: diamondbot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Baiduspider
User-agent: Baiduspider-video
User-agent: Baiduspider-image
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: YoudaoBot
Disallow: /

User-agent: EasouSpider
Disallow: /

User-agent: AhrefsBot
disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: proximic
Disallow: /

User-agent: *
Disallow: /iws/

User-agent: anthropic-ai
#Disallow: /
Crawl-delay: 240

User-agent: ClaudeBot
#Disallow: /
Crawl-delay: 240

User-agent: Applebot
Crawl-delay: 120

User-agent: AwarioBot
Crawl-delay: 120

User-agent: AwarioRssBot
User-agent: AwarioSmartBot
Crawl-delay: 120