Sitemap: https://archiveshub.jisc.ac.uk/sitemap_index.xml

# Block crawling of Microsites as is duplicate content

User-Agent: *
Disallow: /manchesteruniversity/
Disallow: /glaas/
Disallow: /designarchives/
Disallow: /bruneluniversity/
Disallow: /uel/
Disallow: /kingstonuniversity/
Disallow: /universityofportsmouth/
Disallow: /salforduniversity/
Disallow: /rcpsg/
Disallow: /newcastleuniversity/
Disallow: /southamptonuniversity/
Disallow: /hattongallery/

# Bots block and crawl delays last updated 9th Feb 2018
# Block some bots

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Ezooms
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: Vegi bot
Disallow: /

User-agent: Superfeedr
Disallow: /

User-agent: BUbiNG
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Gluten Free Crawler
Disallow: /

# Crawl delay some bots

User-agent: Yandex
Crawl-delay: 10

User-agent: CCBot
Crawl-delay: 10

User-agent: GarlikCrawler
Crawl-delay: 30

User-agent: bingbot
Crawl-delay: 5