# robots.txt for xml.mfd-consult.dk User-agent: * Disallow: /spamtrap/ Disallow: /ws/2003/02/geo/ User-agent: msnbot Disallow: / User-agent: Slurp Disallow: / User-agent: Charlotte Disallow: / User-agent: Yandex Disallow: / User-Agent: MJ12bot Disallow: / Crawl-Delay: 20