# If you would like to crawl CourtListener, please contact us. We also have an
# extensive REST API and provide bulk data.

# Google, AOL, Bing, Yahoo!, DuckDuckGo
# These support meta robots and x-robots-tag or are otherwise harmless (DDG)
# Crawl everything except a few explicit blocks
User-agent: Googlebot
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/rest/v3/

User-agent: bingbot
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/rest/v3/

User-agent: Slurp
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/rest/v3/

User-agent: DuckDuckBot
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/rest/v3/

# Needed for open graph crawling
User-agent: twitterbot
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/rest/v3/

User-agent: facebookexternalhit
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/rest/v3/

# Yandex, Ask
# These support meta robots, but not x-robots-tag
# Crawl everything except real files
User-agent: YandexBot
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/rest/v3/
Disallow: /pdf/
Disallow: /wpd/
Disallow: /txt/
Disallow: /doc/

User-agent: teoma
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/rest/v3/
Disallow: /pdf/
Disallow: /wpd/
Disallow: /txt/
Disallow: /doc/

User-agent: ia_archiver
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/rest/v3/
Disallow: /pdf/
Disallow: /wpd/
Disallow: /txt/
Disallow: /doc/

User-agent: MojeekBot
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/rest/v3/
Disallow: /pdf/
Disallow: /wpd/
Disallow: /txt/
Disallow: /doc/

# Baidu, Blekko, Others
# No support for robots meta tag nor x-robots-tag.
# Be conservative; Block everything.
User-agent: *
Disallow: /

Sitemap: https://www.courtlistener.com/sitemap.xml