# All robots will spider the domain User-agent: * Allow: / User-agent: * Allow: /web/ # Disallow directory /manager/ User-agent: * Disallow: /web/manager/ # Disallow directory /assets/ User-agent: * Disallow: /web/assets/ # Disallow directory /templates/ User-agent: * Disallow: /web/templates/