User-agent: *
####################################
# Geoserver
Disallow: /map/gsrv/gsrv1/
####################################
# GeoNetwork
Allow: /map/catalog/srv/*/catalog.search#/metadata/*

Disallow: /map/catalog/srv/search
Disallow: /map/catalog/srv/search?*
Disallow: /map/catalog/srv/api/records/*
Disallow: /map/catalog/static/api/*
## TODO tbc
Disallow: /map/catalog/api/

####################################
# CKAN
Allow: /catalog/dataset
Disallow: /catalog/dataset?*

Allow: /catalog/dataset/*/resource/*
Disallow: /catalog/dataset/*/history
Disallow: /catalog/dataset/rate/

Allow: /catalog/organization
Disallow: /catalog/organization?*

Disallow: /catalog/api/
Disallow: /catalog/revision/
Disallow: /catalog/stats
Disallow: /catalog/ckan-admin

User-Agent: *
Crawl-Delay: 30

# desidered bots
User-Agent: Bingbot
Crawl-Delay: 20

User-Agent: YandexBot
Crawl-Delay: 60

# BANNED Useless bots
User-Agent: SemrushBot
Disallow: /