diff --git a/conf.py b/conf.py index 4a8b72584e..f7d028c19b 100644 --- a/conf.py +++ b/conf.py @@ -272,7 +272,7 @@ html_static_path = ['docs/static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +html_extra_path = ["docs/robots.txt"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/docs/robots.txt b/docs/robots.txt new file mode 100644 index 0000000000..39a4eaeed9 --- /dev/null +++ b/docs/robots.txt @@ -0,0 +1,6 @@ +User-agent: * +Allow: /*/latest/ +Allow: /en/latest/ # Fallback for bots that don't understand wildcards +Allow: /*/5.3.x/ +Allow: /en/5.3.x/ # Fallback for bots that don't understand wildcards +Disallow: /