From e1a843878e4272334869b1db4a8222ae344f5cb8 Mon Sep 17 00:00:00 2001 From: Dan Halbert Date: Mon, 17 Aug 2020 09:48:17 -0400 Subject: [PATCH] add robots.txt to specify doc versions to appear in search engines --- conf.py | 2 +- docs/robots.txt | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 docs/robots.txt diff --git a/conf.py b/conf.py index 4a8b72584e..f7d028c19b 100644 --- a/conf.py +++ b/conf.py @@ -272,7 +272,7 @@ html_static_path = ['docs/static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +html_extra_path = ["docs/robots.txt"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/docs/robots.txt b/docs/robots.txt new file mode 100644 index 0000000000..39a4eaeed9 --- /dev/null +++ b/docs/robots.txt @@ -0,0 +1,6 @@ +User-agent: * +Allow: /*/latest/ +Allow: /en/latest/ # Fallback for bots that don't understand wildcards +Allow: /*/5.3.x/ +Allow: /en/5.3.x/ # Fallback for bots that don't understand wildcards +Disallow: /