# If you would like to crawl CourtListener, please contact us. We also have an
# extensive REST API and provide bulk data.

# Recently blocked dockets, opinions, and audio files. See:
# https://github.com/freelawproject/courtlistener/issues/1007
Disallow: /docket/16653637/jalal-v-bajic/


Disallow: /opinion/4854791/in-re-andrew-h-ca14/
Disallow: /opinion/4854740/in-re-judiciarys-response-to-the-covid-19-outbreak/
Disallow: /opinion/4854684/in-re-judiciarys-response-to-the-covid-19-outbreak/
Disallow: /opinion/4855201/state-v-lee/
Disallow: /opinion/4855092/simon-v-andrews/
Disallow: /opinion/4854897/borelli-v-renaldi/
Disallow: /opinion/4854895/borelli-v-renaldi/
Disallow: /opinion/4855600/state-v-david/
Disallow: /opinion/4855442/jd-v-pd/
Disallow: /opinion/4855897/in-re-lg/
Disallow: /opinion/4855822/qasimyar-v-maricopa/
Disallow: /opinion/4855662/doetkott-v-secretary-of-health-and-human-services/
Disallow: /opinion/4855647/international-investors-v-town-plan-zoning-commission/





# Google, AOL, Bing, Yahoo!, DuckDuckGo
# These support meta robots and x-robots-tag or are otherwise harmless (DDG)
# Crawl everything except a few explicit blocks
User-agent: Googlebot
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/bulk-data/
Disallow: /static/judge_pics/

User-agent: bingbot
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/bulk-data/
Disallow: /static/judge_pics/

User-agent: Slurp
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/bulk-data/
Disallow: /static/judge_pics/

User-agent: DuckDuckBot
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/bulk-data/
Disallow: /static/judge_pics/

# Needed for open graph crawling
User-agent: twitterbot
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/bulk-data/
Disallow: /static/judge_pics/

User-agent: facebookexternalhit
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/bulk-data/
Disallow: /static/judge_pics/

# Yandex, Ask
# These support meta robots, but not x-robots-tag
# Crawl everything except real files
User-agent: YandexBot
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/bulk-data/
Disallow: /pdf/
Disallow: /wpd/
Disallow: /txt/
Disallow: /doc/
Disallow: /static/judge_pics/

User-agent: teoma
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/bulk-data/
Disallow: /pdf/
Disallow: /wpd/
Disallow: /txt/
Disallow: /doc/
Disallow: /static/judge_pics/

User-agent: ia_archiver
Disallow: /robots.txt
Disallow: /assets/
Disallow: /api/rest/v1/
Disallow: /api/rest/v2/
Disallow: /api/bulk-data/
Disallow: /pdf/
Disallow: /wpd/
Disallow: /txt/
Disallow: /doc/
Disallow: /static/judge_pics/


# Baidu, Blekko, Others
# No support for robots meta tag nor x-robots-tag.
# Be conservative; Block everything.
User-agent: *
Disallow: /

Sitemap: https://www.courtlistener.com/sitemap.xml