User-agent: *
Allow: /
Disallow: /videos
Disallow: /audio
Disallow: /hls
Disallow: /transcripts
Disallow: /data
Disallow: /js
Disallow: /css
Disallow: /scripts
Disallow: /ath/audio
Disallow: /ath/pdf
Disallow: /ath/images
Disallow: /html/ath
# badly-behaved bots
# MJ12Bot was downloading audio - wtf.
User-agent: MJ12Bot
Disallow: /
User-agent: petalbot
Disallow: /
# It's greedy, but it feeds Alexa per the bot web-page, so may as well allow it.
# User-agent: Amazonbot
# Disallow: /