add sitemap to rust docs

src/bootstrap/src/core/build_steps/dist.rs

@@ -82,6 +82,7 @@ fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
         tarball.set_product_name("Rust Documentation");
         tarball.add_bulk_dir(builder.doc_out(host), dest);
         tarball.add_file(builder.src.join("src/doc/robots.txt"), dest, FileType::Regular);
+        tarball.add_file(builder.src.join("src/doc/sitemap.txt"), dest, FileType::Regular);
         Some(tarball.generate())
     }
 }

src/doc/robots.txt

@@ -9,3 +9,5 @@ Disallow: /beta/book/first-edition/
 Disallow: /beta/book/second-edition/
 Disallow: /nightly/book/first-edition/
 Disallow: /nightly/book/second-edition/
+
+Sitemap: https://doc.rust-lang.org/sitemap.txt

src/doc/sitemap.txt

@@ -0,0 +1,4 @@
+https://doc.rust-lang.org/stable/
+https://doc.rust-lang.org/beta/
+https://doc.rust-lang.org/nightly/
+