#Allow commands must precede Disallows for that directory
#Do not add a googlebot-specific user-agent without replicating what is in the universal one - google ignores User-agent:* if there is User-agent: Googlebot

# Allow good bots
User-agent: archive.org_bot
Allow: /

# UChicago legacy
User-agent: *
Disallow: /search/
Disallow: /TestInfo/
Disallow: /Test/
Disallow: /StaffInfo/
Disallow: /staffweb/
Disallow: /dldc/
Disallow: /~chas/
Allow: /cgi-bin/nand/search/stc
Allow: /cgi-bin/nand/search/rosenthal
Allow: /cgi-bin/nand/search/databasefinder
Disallow: /cgi-bin/nand/search/databasefinder?
Disallow: /cgi-bin/
Disallow: /e/keith/
Disallow: /archives/
Disallow: /e/chas/
Disallow: /bus/
Disallow: /e/busecon/macroecon/
Disallow: /phplib/
Disallow: /automail/
Disallow: /ui/

# AI Data Scraper
# https://darkvisitors.com/agents/ai2bot

User-agent: AI2Bot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/applebot-extended

User-agent: Applebot-Extended
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/bytespider

User-agent: Bytespider
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/ccbot

User-agent: CCBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/claudebot

User-agent: ClaudeBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/cohere-training-data-crawler

User-agent: cohere-training-data-crawler
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/cotoyogi

User-agent: Cotoyogi
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/datenbank-crawler

User-agent: Datenbank Crawler
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/diffbot

User-agent: Diffbot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/facebookbot

User-agent: FacebookBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/factset-spyderbot

User-agent: Factset_spyderbot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/google-extended

User-agent: Google-Extended
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/gptbot

User-agent: GPTBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/icc-crawler

User-agent: ICC-Crawler
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/kangaroo-bot

User-agent: Kangaroo Bot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/meta-externalagent

User-agent: Meta-ExternalAgent
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/netestate-imprint-crawler

User-agent: netEstate Imprint Crawler
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/omgili

User-agent: omgili
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/pangubot

User-agent: PanguBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/petalbot

User-agent: PetalBot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/semrushbot-ocob

User-agent: SemrushBot-OCOB
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/timpibot

User-agent: Timpibot
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/webzio-extended

User-agent: Webzio-Extended
Disallow: /