WIP

Author: syphar

Cargo.lock

@@ -144,16 +144,19 @@ struct DeserializedMetadata {
 #[derive(Deserialize)]
 struct DeserializedResolve {
     root: String,
+    #[allow(dead_code)]
     nodes: Vec<DeserializedResolveNode>,
 }
 
 #[derive(Deserialize)]
+#[allow(dead_code)]
 struct DeserializedResolveNode {
     id: String,
     deps: Vec<DeserializedResolveDep>,
 }
 
 #[derive(Deserialize)]
 struct DeserializedResolveDep {
+    #[allow(dead_code)]
     pkg: String,
 }

crates/lib/docs_rs_cargo_metadata/src/metadata.rs

@@ -6,8 +6,10 @@ build = "build.rs"
 
 [dependencies]
 anyhow = { workspace = true }
+chrono = { workspace = true }
 docs_rs_env_vars = { path = "../docs_rs_env_vars" }
 docs_rs_opentelemetry = { path = "../docs_rs_opentelemetry" }
+docs_rs_types = { path = "../docs_rs_types" }
 futures-util = { workspace = true }
 hex = "0.4.3"
 opentelemetry = { workspace = true }

crates/lib/docs_rs_database/Cargo.toml

@@ -0,0 +1,116 @@
+use anyhow::Result;
+use chrono::{DateTime, Utc};
+use futures_util::TryStreamExt as _;
+use serde_json::Value;
+use docs_rs_types::{BuildStatus, CrateId, ReleaseId, Version};
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct Release {
+    pub id: ReleaseId,
+    pub version: Version,
+    #[allow(clippy::doc_overindented_list_items)]
+    /// Aggregated build status of the release.
+    /// * no builds -> build In progress
+    /// * any build is successful -> Success
+    ///   -> even with failed or in-progress builds we have docs to show
+    /// * any build is failed -> Failure
+    ///   -> we can only have Failure or InProgress here, so the Failure is the
+    ///      important part on this aggregation level.
+    /// * the rest is all builds are in-progress -> InProgress
+    ///   -> if we have any builds, and the previous conditions don't match, we end
+    ///      up here, but we still check.
+    ///
+    /// calculated in a database view : `release_build_status`
+    pub build_status: BuildStatus,
+    pub yanked: Option<bool>,
+    pub is_library: Option<bool>,
+    pub rustdoc_status: Option<bool>,
+    pub target_name: Option<String>,
+    pub default_target: Option<String>,
+    pub doc_targets: Option<Vec<String>>,
+    pub release_time: Option<DateTime<Utc>>,
+}
+
+pub fn parse_doc_targets(targets: Value) -> Vec<String> {
+    let mut targets: Vec<String> = serde_json::from_value(targets).unwrap_or_default();
+    targets.sort_unstable();
+    targets
+}
+
+/// Return all releases for a crate, sorted in descending order by semver
+pub async fn releases_for_crate(
+    conn: &mut sqlx::PgConnection,
+    crate_id: CrateId,
+) -> Result<Vec<Release>, anyhow::Error> {
+    let mut releases: Vec<Release> = sqlx::query!(
+        r#"SELECT
+             releases.id as "id: ReleaseId",
+             releases.version as "version: Version",
+             release_build_status.build_status as "build_status!: BuildStatus",
+             releases.yanked,
+             releases.is_library,
+             releases.rustdoc_status,
+             releases.release_time,
+             releases.target_name,
+             releases.default_target,
+             releases.doc_targets
+         FROM releases
+         INNER JOIN release_build_status ON releases.id = release_build_status.rid
+         WHERE
+             releases.crate_id = $1"#,
+        crate_id.0,
+    )
+    .fetch(&mut *conn)
+    .try_filter_map(|row| async move {
+        Ok(Some(Release {
+            id: row.id,
+            version: row.version,
+            build_status: row.build_status,
+            yanked: row.yanked,
+            is_library: row.is_library,
+            rustdoc_status: row.rustdoc_status,
+            target_name: row.target_name,
+            default_target: row.default_target,
+            doc_targets: row.doc_targets.map(parse_doc_targets),
+            release_time: row.release_time,
+        }))
+    })
+    .try_collect()
+    .await?;
+
+    releases.sort_by(|a, b| b.version.cmp(&a.version));
+    Ok(releases)
+}
+
+pub fn latest_release(releases: &[Release]) -> Option<&Release> {
+    if let Some(release) = releases.iter().find(|release| {
+        release.version.pre.is_empty()
+            && release.yanked == Some(false)
+            && release.build_status != BuildStatus::InProgress
+    }) {
+        Some(release)
+    } else {
+        releases
+            .iter()
+            .find(|release| release.build_status != BuildStatus::InProgress)
+    }
+}
+
+pub async fn update_latest_version_id(
+    conn: &mut sqlx::PgConnection,
+    crate_id: CrateId,
+) -> Result<()> {
+    let releases = releases_for_crate(conn, crate_id).await?;
+
+    sqlx::query!(
+        "UPDATE crates
+         SET latest_version_id = $2
+         WHERE id = $1",
+        crate_id.0,
+        latest_release(&releases).map(|release| release.id.0),
+    )
+    .execute(&mut *conn)
+    .await?;
+
+    Ok(())
+}

crates/lib/docs_rs_database/src/crate_details.rs

@@ -1,4 +1,5 @@
 mod config;
+pub mod crate_details;
 mod errors;
 mod metrics;
 mod migrations;

crates/lib/docs_rs_database/src/lib.rs

@@ -10,7 +10,10 @@ use docs_rs::{
         list_crate_priorities, queue_builder, remove_crate_priority, set_crate_priority,
     },
 };
-use docs_rs_database::service_config::{ConfigName, get_config, set_config};
+use docs_rs_database::{
+    crate_details,
+    service_config::{ConfigName, get_config, set_config},
+};
 use docs_rs_storage::add_path_into_database;
 use docs_rs_types::{CrateId, Version};
 use futures_util::StreamExt;
@@ -533,7 +536,7 @@ impl DatabaseSubcommand {
 
                         println!("handling crate {}", row.name);
 
-                        db::update_latest_version_id(&mut update_conn, row.id).await?;
+                        crate_details::update_latest_version_id(&mut update_conn, row.id).await?;
                     }
 
                     Ok::<(), anyhow::Error>(())

src/bin/cratesfyi.rs

@@ -1,6 +1,6 @@
 use crate::{
     BuildPackageSummary, Config, Context, Index, RustwideBuilder,
-    db::{delete_crate, delete_version, update_latest_version_id},
+    db::{delete_crate, delete_version},
     docbuilder::{BuilderMetrics, PackageKind},
     error::Result,
     utils::{get_crate_priority, report_error},
@@ -10,6 +10,7 @@ use chrono::NaiveDate;
 use crates_index_diff::{Change, CrateVersion};
 use docs_rs_database::{
     AsyncPoolClient, Pool,
+    crate_details::update_latest_version_id,
     service_config::{ConfigName, get_config, set_config},
 };
 use docs_rs_fastly::{Cdn, CdnBehaviour as _};

src/build_queue.rs

@@ -1,10 +1,7 @@
-use crate::{
-    docbuilder::DocCoverage,
-    error::Result,
-    web::crate_details::{latest_release, releases_for_crate},
-};
+use crate::{docbuilder::DocCoverage, error::Result};
 use anyhow::{Context, anyhow};
 use docs_rs_cargo_metadata::{MetadataPackage, ReleaseDependencyList};
+use docs_rs_database::crate_details::update_latest_version_id;
 use docs_rs_registry_api::{CrateData, CrateOwner, ReleaseData};
 use docs_rs_storage::CompressionAlgorithm;
 use docs_rs_types::{BuildId, BuildStatus, CrateId, Feature, ReleaseId, Version};
@@ -129,25 +126,6 @@ pub(crate) async fn finish_release(
     Ok(())
 }
 
-pub async fn update_latest_version_id(
-    conn: &mut sqlx::PgConnection,
-    crate_id: CrateId,
-) -> Result<()> {
-    let releases = releases_for_crate(conn, crate_id).await?;
-
-    sqlx::query!(
-        "UPDATE crates
-         SET latest_version_id = $2
-         WHERE id = $1",
-        crate_id.0,
-        latest_release(&releases).map(|release| release.id.0),
-    )
-    .execute(&mut *conn)
-    .await?;
-
-    Ok(())
-}
-
 pub async fn update_build_status(
     conn: &mut sqlx::PgConnection,
     release_id: ReleaseId,

src/db/add_package.rs

@@ -1,12 +1,11 @@
 use crate::{Config, error::Result};
 use anyhow::Context as _;
+use docs_rs_database::crate_details::update_latest_version_id;
 use docs_rs_storage::{AsyncStorage, rustdoc_archive_path, source_archive_path};
 use docs_rs_types::{CrateId, Version};
 use fn_error_context::context;
 use sqlx::Connection;
 
-use super::update_latest_version_id;
-
 /// List of directories in docs.rs's underlying storage (either the database or S3) containing a
 /// subdirectory named after the crate. Those subdirectories will be deleted.
 static LIBRARY_STORAGE_PATHS_TO_DELETE: &[&str] = &["rustdoc", "rustdoc-json", "sources"];

src/db/delete.rs

@@ -1,5 +1,4 @@
 //! Database operations
-pub use self::add_package::update_latest_version_id;
 pub(crate) use self::add_package::{
     add_doc_coverage, finish_build, finish_release, initialize_build, initialize_crate,
     initialize_release, update_build_with_error,