User-agent: * Allow: / # Sitemap location Sitemap: /sitemap.xml # Prevent crawling of redirect URLs and admin Disallow: /out/ Disallow: /admin/ # Crawl-delay for being nice to servers Crawl-delay: 1