diff --git a/public/robots.txt b/public/robots.txt index 8e8a84d320..09de95ffc7 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,3 +1,7 @@ +# Default Settings +User-agent: * + +# Pages that bots are allowed to visit Allow: /$ Allow: /about$ Allow: /accessibility$ @@ -12,9 +16,6 @@ Allow: /opensearch.xml$ Disallow: / -# User-agent: * -# Allow: / - # Search pages: infinite query-param combinations Disallow: /search