From a96bad1631fb3a50bd75730894008ef6b991d6c8 Mon Sep 17 00:00:00 2001
From: Denis Cornehl <denis@cornehl.org>
Date: Fri, 23 May 2025 21:40:09 +0200
Subject: [PATCH] Add rustdoc JSON download endpoints & about-page

---
 src/docbuilder/rustwide_builder.rs     |   9 +-
 src/storage/mod.rs                     |  36 ++++-
 src/web/error.rs                       |  10 ++
 src/web/routes.rs                      |  16 ++
 src/web/rustdoc.rs                     | 197 ++++++++++++++++++++++++-
 src/web/sitemap.rs                     |   5 +
 templates/about-base.html              |   4 +
 templates/core/about/rustdoc-json.html |  65 ++++++++
 8 files changed, 334 insertions(+), 8 deletions(-)
 create mode 100644 templates/core/about/rustdoc-json.html

diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs
index cfe72d7f8..53ae91d57 100644
--- a/src/docbuilder/rustwide_builder.rs
+++ b/src/docbuilder/rustwide_builder.rs
@@ -1494,12 +1494,11 @@ mod tests {
                         .map(|f| f.strip_prefix(&json_prefix).unwrap().to_owned())
                         .collect();
                     json_files.sort();
+                    assert!(json_files[0].starts_with(&format!("empty-library_1.0.0_{target}_")));
+                    assert!(json_files[0].ends_with(".json.zst"));
                     assert_eq!(
-                        json_files,
-                        vec![
-                            format!("empty-library_1.0.0_{target}_45.json.zst"),
-                            format!("empty-library_1.0.0_{target}_latest.json.zst"),
-                        ]
+                        json_files[1],
+                        format!("empty-library_1.0.0_{target}_latest.json.zst")
                     );
 
                     if target == &default_target {
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index b97a9f802..13a2b3c27 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -22,14 +22,16 @@ use fn_error_context::context;
 use futures_util::stream::BoxStream;
 use mime::Mime;
 use path_slash::PathExt;
-use std::iter;
+use serde_with::{DeserializeFromStr, SerializeDisplay};
 use std::{
     fmt, fs,
     io::{self, BufReader},
+    num::ParseIntError,
     ops::RangeInclusive,
     path::{Path, PathBuf},
     sync::Arc,
 };
+use std::{iter, str::FromStr};
 use tokio::{io::AsyncWriteExt, runtime::Runtime};
 use tracing::{error, info_span, instrument, trace};
 use walkdir::WalkDir;
@@ -815,7 +817,7 @@ pub(crate) fn rustdoc_archive_path(name: &str, version: &str) -> String {
     format!("rustdoc/{name}/{version}.zip")
 }
 
-#[derive(strum::Display, Debug, PartialEq, Eq)]
+#[derive(strum::Display, Debug, PartialEq, Eq, Clone, SerializeDisplay, DeserializeFromStr)]
 #[strum(serialize_all = "snake_case")]
 pub(crate) enum RustdocJsonFormatVersion {
     #[strum(serialize = "{0}")]
@@ -823,6 +825,17 @@ pub(crate) enum RustdocJsonFormatVersion {
     Latest,
 }
 
+impl FromStr for RustdocJsonFormatVersion {
+    type Err = ParseIntError;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        if s == "latest" {
+            Ok(RustdocJsonFormatVersion::Latest)
+        } else {
+            s.parse::<u16>().map(RustdocJsonFormatVersion::Version)
+        }
+    }
+}
+
 pub(crate) fn rustdoc_json_path(
     name: &str,
     version: &str,
@@ -842,6 +855,25 @@ pub(crate) fn source_archive_path(name: &str, version: &str) -> String {
 mod test {
     use super::*;
     use std::env;
+    use test_case::test_case;
+
+    #[test_case("latest", RustdocJsonFormatVersion::Latest)]
+    #[test_case("42", RustdocJsonFormatVersion::Version(42))]
+    fn test_json_format_version(input: &str, expected: RustdocJsonFormatVersion) {
+        // test Display
+        assert_eq!(expected.to_string(), input);
+        // test FromStr
+        assert_eq!(expected, input.parse().unwrap());
+
+        let json_input = format!("\"{input}\"");
+        // test Serialize
+        assert_eq!(serde_json::to_string(&expected).unwrap(), json_input);
+        // test Deserialize
+        assert_eq!(
+            serde_json::from_str::<RustdocJsonFormatVersion>(&json_input).unwrap(),
+            expected
+        );
+    }
 
     #[test]
     fn test_get_file_list() -> Result<()> {
diff --git a/src/web/error.rs b/src/web/error.rs
index 8b06b9604..7930367a6 100644
--- a/src/web/error.rs
+++ b/src/web/error.rs
@@ -44,6 +44,8 @@ pub enum AxumNope {
     OwnerNotFound,
     #[error("Requested crate does not have specified version")]
     VersionNotFound,
+    #[error("Requested release doesn't have docs for the given target")]
+    TargetNotFound,
     #[error("Search yielded no results")]
     NoResults,
     #[error("Unauthorized: {0}")]
@@ -77,6 +79,14 @@ impl AxumNope {
                 message: "no such build".into(),
                 status: StatusCode::NOT_FOUND,
             },
+            AxumNope::TargetNotFound => {
+                // user tried to navigate to a target that doesn't exist
+                ErrorInfo {
+                    title: "The requested target does not exist",
+                    message: "no such target".into(),
+                    status: StatusCode::NOT_FOUND,
+                }
+            }
             AxumNope::CrateNotFound => {
                 // user tried to navigate to a crate that doesn't exist
                 // TODO: Display the attempted crate and a link to a search for said crate
diff --git a/src/web/routes.rs b/src/web/routes.rs
index 39a59e9a2..8257d4630 100644
--- a/src/web/routes.rs
+++ b/src/web/routes.rs
@@ -301,10 +301,26 @@ pub(super) fn build_axum_routes() -> AxumRouter {
             "/crate/{name}/{version}/download",
             get_internal(super::rustdoc::download_handler),
         )
+        .route_with_tsr(
+            "/crate/{name}/{version}/json",
+            get_internal(super::rustdoc::json_download_handler),
+        )
+        .route_with_tsr(
+            "/crate/{name}/{version}/json/{format_version}",
+            get_internal(super::rustdoc::json_download_handler),
+        )
         .route(
             "/crate/{name}/{version}/target-redirect/{*path}",
             get_internal(super::rustdoc::target_redirect_handler),
         )
+        .route_with_tsr(
+            "/crate/{name}/{version}/{target}/json",
+            get_internal(super::rustdoc::json_download_handler),
+        )
+        .route_with_tsr(
+            "/crate/{name}/{version}/{target}/json/{format_version}",
+            get_internal(super::rustdoc::json_download_handler),
+        )
         .route(
             "/{name}/badge.svg",
             get_internal(super::rustdoc::badge_handler),
diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs
index a725e706a..8c443e939 100644
--- a/src/web/rustdoc.rs
+++ b/src/web/rustdoc.rs
@@ -3,7 +3,7 @@
 use crate::{
     AsyncStorage, Config, InstanceMetrics, RUSTDOC_STATIC_STORAGE_PREFIX,
     db::Pool,
-    storage::rustdoc_archive_path,
+    storage::{RustdocJsonFormatVersion, rustdoc_archive_path, rustdoc_json_path},
     utils,
     web::{
         MetaData, ReqVersion, axum_cached_redirect, axum_parse_uri_with_params,
@@ -817,6 +817,72 @@ pub(crate) async fn badge_handler(
     ))
 }
 
+#[derive(Clone, Deserialize, Debug)]
+pub(crate) struct JsonDownloadParams {
+    pub(crate) name: String,
+    pub(crate) version: ReqVersion,
+    pub(crate) target: Option<String>,
+    pub(crate) format_version: Option<RustdocJsonFormatVersion>,
+}
+
+#[instrument(skip_all)]
+pub(crate) async fn json_download_handler(
+    Path(params): Path<JsonDownloadParams>,
+    mut conn: DbConnection,
+    Extension(config): Extension<Arc<Config>>,
+) -> AxumResult<impl IntoResponse> {
+    let matched_release = match_version(&mut conn, &params.name, &params.version)
+        .await?
+        .assume_exact_name()?;
+
+    if !matched_release.rustdoc_status() {
+        // without docs we'll never have JSON docs too
+        return Err(AxumNope::ResourceNotFound);
+    }
+
+    let krate = CrateDetails::from_matched_release(&mut conn, matched_release).await?;
+
+    let target = if let Some(wanted_target) = params.target {
+        if krate
+            .metadata
+            .doc_targets
+            .as_ref()
+            .expect("we are checking rustdoc_status() above, so we always have metadata")
+            .iter()
+            .any(|s| s == &wanted_target)
+        {
+            wanted_target
+        } else {
+            return Err(AxumNope::TargetNotFound);
+        }
+    } else {
+        krate
+            .metadata
+            .default_target
+            .as_ref()
+            .expect("we are checking rustdoc_status() above, so we always have metadata")
+            .to_string()
+    };
+
+    let format_version = params
+        .format_version
+        .unwrap_or(RustdocJsonFormatVersion::Latest);
+
+    let storage_path = rustdoc_json_path(
+        &krate.name,
+        &krate.version.to_string(),
+        &target,
+        format_version,
+    );
+
+    // since we didn't build rustdoc json for all releases yet,
+    // this redirect might redirect to a location that doesn't exist.
+    Ok(super::axum_cached_redirect(
+        format!("{}/{}", config.s3_static_root_path, storage_path),
+        CachePolicy::ForeverInCdn,
+    )?)
+}
+
 #[instrument(skip_all)]
 pub(crate) async fn download_handler(
     Path((name, req_version)): Path<(String, ReqVersion)>,
@@ -874,6 +940,7 @@ pub(crate) async fn static_asset_handler(
 
 #[cfg(test)]
 mod test {
+    use super::*;
     use crate::{
         Config,
         registry_api::{CrateOwner, OwnerKind},
@@ -3009,4 +3076,132 @@ mod test {
             Ok(())
         });
     }
+
+    #[test_case(
+        "latest/json",
+        "0.2.0",
+        "x86_64-unknown-linux-gnu",
+        RustdocJsonFormatVersion::Latest
+    )]
+    #[test_case(
+        "0.1/json",
+        "0.1.0",
+        "x86_64-unknown-linux-gnu",
+        RustdocJsonFormatVersion::Latest;
+        "semver"
+    )]
+    #[test_case(
+        "0.1.0/json",
+        "0.1.0",
+        "x86_64-unknown-linux-gnu",
+        RustdocJsonFormatVersion::Latest
+    )]
+    #[test_case(
+        "latest/json/latest",
+        "0.2.0",
+        "x86_64-unknown-linux-gnu",
+        RustdocJsonFormatVersion::Latest
+    )]
+    #[test_case(
+        "latest/json/42",
+        "0.2.0",
+        "x86_64-unknown-linux-gnu",
+        RustdocJsonFormatVersion::Version(42)
+    )]
+    #[test_case(
+        "latest/i686-pc-windows-msvc/json",
+        "0.2.0",
+        "i686-pc-windows-msvc",
+        RustdocJsonFormatVersion::Latest
+    )]
+    #[test_case(
+        "latest/i686-pc-windows-msvc/json/42",
+        "0.2.0",
+        "i686-pc-windows-msvc",
+        RustdocJsonFormatVersion::Version(42)
+    )]
+    fn json_download(
+        request_path_suffix: &str,
+        redirect_version: &str,
+        redirect_target: &str,
+        redirect_format_version: RustdocJsonFormatVersion,
+    ) {
+        async_wrapper(|env| async move {
+            env.override_config(|config| {
+                config.s3_static_root_path = "https://static.docs.rs".into();
+            });
+            env.fake_release()
+                .await
+                .name("dummy")
+                .version("0.1.0")
+                .archive_storage(true)
+                .default_target("x86_64-unknown-linux-gnu")
+                .add_target("i686-pc-windows-msvc")
+                .create()
+                .await?;
+
+            env.fake_release()
+                .await
+                .name("dummy")
+                .version("0.2.0")
+                .archive_storage(true)
+                .default_target("x86_64-unknown-linux-gnu")
+                .add_target("i686-pc-windows-msvc")
+                .create()
+                .await?;
+
+            let web = env.web_app().await;
+
+            web.assert_redirect_cached_unchecked(
+                &format!("/crate/dummy/{request_path_suffix}"),
+                &format!("https://static.docs.rs/rustdoc-json/dummy/{redirect_version}/{redirect_target}/\
+                    dummy_{redirect_version}_{redirect_target}_{redirect_format_version}.json.zst"),
+                CachePolicy::ForeverInCdn,
+                &env.config(),
+            )
+            .await?;
+            Ok(())
+        });
+    }
+
+    #[test_case("0.1.0/json"; "rustdoc status false")]
+    #[test_case("0.2.0/unknown-target/json"; "unknown target")]
+    #[test_case("0.42.0/json"; "unknown version")]
+    fn json_download_not_found(request_path_suffix: &str) {
+        async_wrapper(|env| async move {
+            env.override_config(|config| {
+                config.s3_static_root_path = "https://static.docs.rs".into();
+            });
+
+            env.fake_release()
+                .await
+                .name("dummy")
+                .version("0.1.0")
+                .archive_storage(true)
+                .default_target("x86_64-unknown-linux-gnu")
+                .add_target("i686-pc-windows-msvc")
+                .binary(true) // binary => rustdoc_status = false
+                .create()
+                .await?;
+
+            env.fake_release()
+                .await
+                .name("dummy")
+                .version("0.2.0")
+                .archive_storage(true)
+                .default_target("x86_64-unknown-linux-gnu")
+                .add_target("i686-pc-windows-msvc")
+                .create()
+                .await?;
+
+            let web = env.web_app().await;
+
+            let response = web
+                .get(&format!("/crate/dummy/{request_path_suffix}"))
+                .await?;
+
+            assert_eq!(response.status(), StatusCode::NOT_FOUND);
+            Ok(())
+        });
+    }
 }
diff --git a/src/web/sitemap.rs b/src/web/sitemap.rs
index 72c497f11..bb704aabd 100644
--- a/src/web/sitemap.rs
+++ b/src/web/sitemap.rs
@@ -142,6 +142,7 @@ about_page!(AboutPageBadges, "core/about/badges.html");
 about_page!(AboutPageMetadata, "core/about/metadata.html");
 about_page!(AboutPageRedirection, "core/about/redirections.html");
 about_page!(AboutPageDownload, "core/about/download.html");
+about_page!(AboutPageRustdocJson, "core/about/rustdoc-json.html");
 
 pub(crate) async fn about_handler(subpage: Option<Path<String>>) -> AxumResult<impl IntoResponse> {
     let subpage = match subpage {
@@ -170,6 +171,10 @@ pub(crate) async fn about_handler(subpage: Option<Path<String>>) -> AxumResult<i
             active_tab: "download",
         }
         .into_response(),
+        "rustdoc-json" => AboutPageRustdocJson {
+            active_tab: "rustdoc-json",
+        }
+        .into_response(),
         _ => {
             let msg = "This /about page does not exist. \
                 Perhaps you are interested in <a href=\"https://github.com/rust-lang/docs.rs/tree/master/templates/core/about\">creating</a> it?";
diff --git a/templates/about-base.html b/templates/about-base.html
index c3407ffa2..8eb4ed0b9 100644
--- a/templates/about-base.html
+++ b/templates/about-base.html
@@ -34,6 +34,10 @@ <h1 id="crate-title" class="no-description">Docs.rs documentation</h1>
                         {% set text = crate::icons::IconDownload.render_solid(false, false, "") %}
                         {% set text = "{} <span class='title'>Download</span>"|format(text) %}
                         {% call macros::active_link(expected="download", href="/about/download", text=text) %}
+
+                        {% set text = crate::icons::IconFileCode.render_solid(false, false, "") %}
+                        {% set text = "{} <span class='title'>Rustdoc JSON</span>"|format(text) %}
+                        {% call macros::active_link(expected="rustdoc-json", href="/about/rustdoc-json", text=text) %}
                     </ul>
                 </div>
             </div>
diff --git a/templates/core/about/rustdoc-json.html b/templates/core/about/rustdoc-json.html
new file mode 100644
index 000000000..e93e359c9
--- /dev/null
+++ b/templates/core/about/rustdoc-json.html
@@ -0,0 +1,65 @@
+{% extends "about-base.html" %}
+
+{%- block title -%} Rustdoc JSON {%- endblock title -%}
+
+{%- block body -%}
+    <h1>Rustdoc JSON</h1>
+
+    <div class="about-page">
+        <div class="container pure-u-5-6 about">
+            <p>
+                docs.rs builds & hosts the <a href="https://doc.rust-lang.org/rustdoc/unstable-features.html#json-output"><code>rustdoc</code> JSON</a> output.
+            </p>
+            <p>
+                This structured version of the documentation can be used for inspecting the docs
+                and types of the crate in a programmatic way.
+            </p>
+            <p>
+                The JSON file you're downloading might have been built with an older version of <code>rustdoc</code>
+                so you might have to check the <code>format_version</code> attribute to determine how to parse the structure.
+                The <a href="https://docs.rs/rustdoc-types/"><code>rustdoc-types</code></a> crate helps with this.
+            </p>
+            <p><b>
+                We started building rustdoc JSON on 2025-05-23. So any release before that won't have a download
+                available until our rebuilds reached that release.
+            </b></p>
+            <h2>URLs</h2>
+            <p>
+                The download URLs can be generic or specific, depending on what you need. In case of rebuilds we
+                also keep old format versions around. The endpoints <i>redirect</i> to a download URL on
+                <code>https://static.docs.rs</code>. <b>Until we rebuilt all releases, the redirect target might
+                not exist.</b>
+            </p>
+            <p>
+                Here some URL examples you can use.
+            </p>
+            <table class="pure-table pure-table-horizontal">
+                <thead>
+                    <tr>
+                        <th>URL</th>
+                        <th>Redirects to JSON download of</th>
+                    </tr>
+                </thead>
+
+                <tbody>
+                    <tr>
+                        <td><a href="/crate/clap/latest/json">https://docs.rs/crate/clap/latest/json</a></td>
+                        <td>latest version, default target, latest format-version</td>
+                    </tr>
+                    <tr>
+                        <td><a href="/crate/clap/latest/json/42">https://docs.rs/crate/clap/latest/json/42</a></td>
+                        <td>latest version, default target, format-version 42</td>
+                    </tr>
+                    <tr>
+                        <td><a href="/crate/clap/%7E4/json">https://docs.rs/crate/clap/~4/json</a></td>
+                        <td>latest v4 via semver, default target, latest format-version</td>
+                    </tr>
+                    <tr>
+                        <td><a href="/crate/clap/latest/i686-pc-windows-msvc/json">https://docs.rs/crate/clap/latest/i686-pc-windows-msvc/json</a></td>
+                        <td>latest version, windows target, latest format-version</td>
+                    </tr>
+                </tbody>
+            </table>
+        </div>
+    </div>
+{%- endblock body %}