diff --git a/Cargo.lock b/Cargo.lock index 27cb7275d..215a1cb74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2003,8 +2003,10 @@ name = "docs_rs_database" version = "0.1.0" dependencies = [ "anyhow", + "chrono", "docs_rs_env_vars", "docs_rs_opentelemetry", + "docs_rs_types", "futures-util", "hex", "opentelemetry", diff --git a/crates/lib/docs_rs_cargo_metadata/src/metadata.rs b/crates/lib/docs_rs_cargo_metadata/src/metadata.rs index 347f3dd05..e9b2b3c52 100644 --- a/crates/lib/docs_rs_cargo_metadata/src/metadata.rs +++ b/crates/lib/docs_rs_cargo_metadata/src/metadata.rs @@ -144,16 +144,4 @@ struct DeserializedMetadata { #[derive(Deserialize)] struct DeserializedResolve { root: String, - nodes: Vec, -} - -#[derive(Deserialize)] -struct DeserializedResolveNode { - id: String, - deps: Vec, -} - -#[derive(Deserialize)] -struct DeserializedResolveDep { - pkg: String, } diff --git a/crates/lib/docs_rs_database/.sqlx/query-1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6.json b/crates/lib/docs_rs_database/.sqlx/query-1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6.json new file mode 100644 index 000000000..e1bcf20ab --- /dev/null +++ b/crates/lib/docs_rs_database/.sqlx/query-1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE crates\n SET latest_version_id = $2\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6" +} diff --git a/crates/lib/docs_rs_database/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json b/crates/lib/docs_rs_database/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json new file mode 100644 index 000000000..ca0a44f4b --- /dev/null +++ b/crates/lib/docs_rs_database/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json @@ -0,0 +1,87 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n releases.id as \"id: ReleaseId\",\n releases.version as \"version: Version\",\n release_build_status.build_status as \"build_status!: BuildStatus\",\n releases.yanked,\n releases.is_library,\n releases.rustdoc_status,\n releases.release_time,\n releases.target_name,\n releases.default_target,\n releases.doc_targets\n FROM releases\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n WHERE\n releases.crate_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id: ReleaseId", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "version: Version", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "build_status!: BuildStatus", + "type_info": { + "Custom": { + "name": "build_status", + "kind": { + "Enum": [ + "in_progress", + "success", + "failure" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "yanked", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "is_library", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "rustdoc_status", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "release_time", + "type_info": "Timestamptz" + }, + { + "ordinal": 7, + "name": "target_name", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "default_target", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "doc_targets", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true + ] + }, + "hash": "4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84" +} diff --git a/crates/lib/docs_rs_database/Cargo.toml b/crates/lib/docs_rs_database/Cargo.toml index 594d8c60c..d83b16c7c 100644 --- a/crates/lib/docs_rs_database/Cargo.toml +++ b/crates/lib/docs_rs_database/Cargo.toml @@ -6,8 +6,10 @@ build = "build.rs" [dependencies] anyhow = { workspace = true } +chrono = { workspace = true } docs_rs_env_vars = { path = "../docs_rs_env_vars" } docs_rs_opentelemetry = { path = "../docs_rs_opentelemetry" } +docs_rs_types = { path = "../docs_rs_types" } futures-util = { workspace = true } hex = "0.4.3" opentelemetry = { workspace = true } diff --git a/crates/lib/docs_rs_database/src/crate_details.rs b/crates/lib/docs_rs_database/src/crate_details.rs new file mode 100644 index 000000000..e591ddf7d --- /dev/null +++ b/crates/lib/docs_rs_database/src/crate_details.rs @@ -0,0 +1,116 @@ +use anyhow::Result; +use chrono::{DateTime, Utc}; +use docs_rs_types::{BuildStatus, CrateId, ReleaseId, Version}; +use futures_util::TryStreamExt as _; +use serde_json::Value; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct Release { + pub id: ReleaseId, + pub version: Version, + #[allow(clippy::doc_overindented_list_items)] + /// Aggregated build status of the release. + /// * no builds -> build In progress + /// * any build is successful -> Success + /// -> even with failed or in-progress builds we have docs to show + /// * any build is failed -> Failure + /// -> we can only have Failure or InProgress here, so the Failure is the + /// important part on this aggregation level. + /// * the rest is all builds are in-progress -> InProgress + /// -> if we have any builds, and the previous conditions don't match, we end + /// up here, but we still check. + /// + /// calculated in a database view : `release_build_status` + pub build_status: BuildStatus, + pub yanked: Option, + pub is_library: Option, + pub rustdoc_status: Option, + pub target_name: Option, + pub default_target: Option, + pub doc_targets: Option>, + pub release_time: Option>, +} + +pub fn parse_doc_targets(targets: Value) -> Vec { + let mut targets: Vec = serde_json::from_value(targets).unwrap_or_default(); + targets.sort_unstable(); + targets +} + +/// Return all releases for a crate, sorted in descending order by semver +pub async fn releases_for_crate( + conn: &mut sqlx::PgConnection, + crate_id: CrateId, +) -> Result, anyhow::Error> { + let mut releases: Vec = sqlx::query!( + r#"SELECT + releases.id as "id: ReleaseId", + releases.version as "version: Version", + release_build_status.build_status as "build_status!: BuildStatus", + releases.yanked, + releases.is_library, + releases.rustdoc_status, + releases.release_time, + releases.target_name, + releases.default_target, + releases.doc_targets + FROM releases + INNER JOIN release_build_status ON releases.id = release_build_status.rid + WHERE + releases.crate_id = $1"#, + crate_id.0, + ) + .fetch(&mut *conn) + .try_filter_map(|row| async move { + Ok(Some(Release { + id: row.id, + version: row.version, + build_status: row.build_status, + yanked: row.yanked, + is_library: row.is_library, + rustdoc_status: row.rustdoc_status, + target_name: row.target_name, + default_target: row.default_target, + doc_targets: row.doc_targets.map(parse_doc_targets), + release_time: row.release_time, + })) + }) + .try_collect() + .await?; + + releases.sort_by(|a, b| b.version.cmp(&a.version)); + Ok(releases) +} + +pub fn latest_release(releases: &[Release]) -> Option<&Release> { + if let Some(release) = releases.iter().find(|release| { + release.version.pre.is_empty() + && release.yanked == Some(false) + && release.build_status != BuildStatus::InProgress + }) { + Some(release) + } else { + releases + .iter() + .find(|release| release.build_status != BuildStatus::InProgress) + } +} + +pub async fn update_latest_version_id( + conn: &mut sqlx::PgConnection, + crate_id: CrateId, +) -> Result<()> { + let releases = releases_for_crate(conn, crate_id).await?; + + sqlx::query!( + "UPDATE crates + SET latest_version_id = $2 + WHERE id = $1", + crate_id.0, + latest_release(&releases).map(|release| release.id.0), + ) + .execute(&mut *conn) + .await?; + + Ok(()) +} diff --git a/crates/lib/docs_rs_database/src/lib.rs b/crates/lib/docs_rs_database/src/lib.rs index 568814d16..5078e8a73 100644 --- a/crates/lib/docs_rs_database/src/lib.rs +++ b/crates/lib/docs_rs_database/src/lib.rs @@ -1,4 +1,5 @@ mod config; +pub mod crate_details; mod errors; mod metrics; mod migrations; diff --git a/crates/lib/docs_rs_repository_stats/.sqlx/query-1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6.json b/crates/lib/docs_rs_repository_stats/.sqlx/query-1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6.json new file mode 100644 index 000000000..e1bcf20ab --- /dev/null +++ b/crates/lib/docs_rs_repository_stats/.sqlx/query-1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE crates\n SET latest_version_id = $2\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6" +} diff --git a/crates/lib/docs_rs_repository_stats/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json b/crates/lib/docs_rs_repository_stats/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json new file mode 100644 index 000000000..ca0a44f4b --- /dev/null +++ b/crates/lib/docs_rs_repository_stats/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json @@ -0,0 +1,87 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n releases.id as \"id: ReleaseId\",\n releases.version as \"version: Version\",\n release_build_status.build_status as \"build_status!: BuildStatus\",\n releases.yanked,\n releases.is_library,\n releases.rustdoc_status,\n releases.release_time,\n releases.target_name,\n releases.default_target,\n releases.doc_targets\n FROM releases\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n WHERE\n releases.crate_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id: ReleaseId", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "version: Version", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "build_status!: BuildStatus", + "type_info": { + "Custom": { + "name": "build_status", + "kind": { + "Enum": [ + "in_progress", + "success", + "failure" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "yanked", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "is_library", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "rustdoc_status", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "release_time", + "type_info": "Timestamptz" + }, + { + "ordinal": 7, + "name": "target_name", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "default_target", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "doc_targets", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true + ] + }, + "hash": "4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84" +} diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index b00ef4234..c1a35e641 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -10,7 +10,10 @@ use docs_rs::{ list_crate_priorities, queue_builder, remove_crate_priority, set_crate_priority, }, }; -use docs_rs_database::service_config::{ConfigName, get_config, set_config}; +use docs_rs_database::{ + crate_details, + service_config::{ConfigName, get_config, set_config}, +}; use docs_rs_storage::add_path_into_database; use docs_rs_types::{CrateId, Version}; use futures_util::StreamExt; @@ -533,7 +536,7 @@ impl DatabaseSubcommand { println!("handling crate {}", row.name); - db::update_latest_version_id(&mut update_conn, row.id).await?; + crate_details::update_latest_version_id(&mut update_conn, row.id).await?; } Ok::<(), anyhow::Error>(()) diff --git a/src/build_queue.rs b/src/build_queue.rs index f42999779..804167865 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -1,6 +1,6 @@ use crate::{ BuildPackageSummary, Config, Context, Index, RustwideBuilder, - db::{delete_crate, delete_version, update_latest_version_id}, + db::{delete_crate, delete_version}, docbuilder::{BuilderMetrics, PackageKind}, error::Result, utils::{get_crate_priority, report_error}, @@ -10,6 +10,7 @@ use chrono::NaiveDate; use crates_index_diff::{Change, CrateVersion}; use docs_rs_database::{ AsyncPoolClient, Pool, + crate_details::update_latest_version_id, service_config::{ConfigName, get_config, set_config}, }; use docs_rs_fastly::{Cdn, CdnBehaviour as _}; diff --git a/src/db/add_package.rs b/src/db/add_package.rs index fe57fa4c0..4432a86fd 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -1,10 +1,7 @@ -use crate::{ - docbuilder::DocCoverage, - error::Result, - web::crate_details::{latest_release, releases_for_crate}, -}; +use crate::{docbuilder::DocCoverage, error::Result}; use anyhow::{Context, anyhow}; use docs_rs_cargo_metadata::{MetadataPackage, ReleaseDependencyList}; +use docs_rs_database::crate_details::update_latest_version_id; use docs_rs_registry_api::{CrateData, CrateOwner, ReleaseData}; use docs_rs_storage::CompressionAlgorithm; use docs_rs_types::{BuildId, BuildStatus, CrateId, Feature, ReleaseId, Version}; @@ -129,25 +126,6 @@ pub(crate) async fn finish_release( Ok(()) } -pub async fn update_latest_version_id( - conn: &mut sqlx::PgConnection, - crate_id: CrateId, -) -> Result<()> { - let releases = releases_for_crate(conn, crate_id).await?; - - sqlx::query!( - "UPDATE crates - SET latest_version_id = $2 - WHERE id = $1", - crate_id.0, - latest_release(&releases).map(|release| release.id.0), - ) - .execute(&mut *conn) - .await?; - - Ok(()) -} - pub async fn update_build_status( conn: &mut sqlx::PgConnection, release_id: ReleaseId, diff --git a/src/db/delete.rs b/src/db/delete.rs index 2674a9009..f16e2f37b 100644 --- a/src/db/delete.rs +++ b/src/db/delete.rs @@ -1,12 +1,11 @@ use crate::{Config, error::Result}; use anyhow::Context as _; +use docs_rs_database::crate_details::update_latest_version_id; use docs_rs_storage::{AsyncStorage, rustdoc_archive_path, source_archive_path}; use docs_rs_types::{CrateId, Version}; use fn_error_context::context; use sqlx::Connection; -use super::update_latest_version_id; - /// List of directories in docs.rs's underlying storage (either the database or S3) containing a /// subdirectory named after the crate. Those subdirectories will be deleted. static LIBRARY_STORAGE_PATHS_TO_DELETE: &[&str] = &["rustdoc", "rustdoc-json", "sources"]; diff --git a/src/db/mod.rs b/src/db/mod.rs index 33a900fbc..715c7ebc9 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,5 +1,4 @@ //! Database operations -pub use self::add_package::update_latest_version_id; pub(crate) use self::add_package::{ add_doc_coverage, finish_build, finish_release, initialize_build, initialize_crate, initialize_release, update_build_with_error, diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 04bb7db88..fd7c9cbab 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -20,6 +20,7 @@ use axum::{ }; use chrono::{DateTime, Utc}; use docs_rs_cargo_metadata::{Dependency, ReleaseDependencyList}; +use docs_rs_database::crate_details::{Release, latest_release, parse_doc_targets}; use docs_rs_headers::CanonicalUrl; use docs_rs_registry_api::OwnerKind; use docs_rs_storage::{AsyncStorage, PathNotFoundError}; @@ -77,33 +78,6 @@ struct RepositoryMetadata { name: Option, } -#[derive(Debug, Clone, Eq, PartialEq)] -pub(crate) struct Release { - pub id: ReleaseId, - pub version: Version, - #[allow(clippy::doc_overindented_list_items)] - /// Aggregated build status of the release. - /// * no builds -> build In progress - /// * any build is successful -> Success - /// -> even with failed or in-progress builds we have docs to show - /// * any build is failed -> Failure - /// -> we can only have Failure or InProgress here, so the Failure is the - /// important part on this aggregation level. - /// * the rest is all builds are in-progress -> InProgress - /// -> if we have any builds, and the previous conditions don't match, we end - /// up here, but we still check. - /// - /// calculated in a database view : `release_build_status` - pub build_status: BuildStatus, - pub yanked: Option, - pub is_library: Option, - pub rustdoc_status: Option, - pub target_name: Option, - pub default_target: Option, - pub doc_targets: Option>, - pub release_time: Option>, -} - impl CrateDetails { #[tracing::instrument(skip(conn))] pub(crate) async fn from_matched_release( @@ -211,7 +185,7 @@ impl CrateDetails { rustdoc_status: krate.rustdoc_status, target_name: krate.target_name.clone(), default_target: krate.default_target, - doc_targets: krate.doc_targets.map(MetaData::parse_doc_targets), + doc_targets: krate.doc_targets.map(parse_doc_targets), yanked: krate.yanked, rustdoc_css_file: krate .rustc_version @@ -370,65 +344,6 @@ impl CrateDetails { } } -pub(crate) fn latest_release(releases: &[Release]) -> Option<&Release> { - if let Some(release) = releases.iter().find(|release| { - release.version.pre.is_empty() - && release.yanked == Some(false) - && release.build_status != BuildStatus::InProgress - }) { - Some(release) - } else { - releases - .iter() - .find(|release| release.build_status != BuildStatus::InProgress) - } -} - -/// Return all releases for a crate, sorted in descending order by semver -pub(crate) async fn releases_for_crate( - conn: &mut sqlx::PgConnection, - crate_id: CrateId, -) -> Result, anyhow::Error> { - let mut releases: Vec = sqlx::query!( - r#"SELECT - releases.id as "id: ReleaseId", - releases.version as "version: Version", - release_build_status.build_status as "build_status!: BuildStatus", - releases.yanked, - releases.is_library, - releases.rustdoc_status, - releases.release_time, - releases.target_name, - releases.default_target, - releases.doc_targets - FROM releases - INNER JOIN release_build_status ON releases.id = release_build_status.rid - WHERE - releases.crate_id = $1"#, - crate_id.0, - ) - .fetch(&mut *conn) - .try_filter_map(|row| async move { - Ok(Some(Release { - id: row.id, - version: row.version, - build_status: row.build_status, - yanked: row.yanked, - is_library: row.is_library, - rustdoc_status: row.rustdoc_status, - target_name: row.target_name, - default_target: row.default_target, - doc_targets: row.doc_targets.map(MetaData::parse_doc_targets), - release_time: row.release_time, - })) - }) - .try_collect() - .await?; - - releases.sort_by(|a, b| b.version.cmp(&a.version)); - Ok(releases) -} - #[derive(Debug, Clone, Template)] #[template(path = "crate/details.html")] struct CrateDetailsPage { @@ -727,7 +642,7 @@ mod tests { fake_release_that_failed_before_build, }; use anyhow::Error; - use docs_rs_database::testing::TestDatabase; + use docs_rs_database::{crate_details::releases_for_crate, testing::TestDatabase}; use docs_rs_registry_api::CrateOwner; use docs_rs_types::KrateName; use http::StatusCode; diff --git a/src/web/mod.rs b/src/web/mod.rs index 3213cee98..d64399d4e 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -25,7 +25,6 @@ mod status; use crate::{ Context, impl_axum_webpage, web::{ - crate_details::Release, metrics::WebMetrics, page::templates::{RenderBrands, RenderSolid, filters}, }, @@ -42,13 +41,13 @@ use axum::{ }; use axum_extra::middleware::option_layer; use chrono::{DateTime, NaiveDate, Utc}; +use docs_rs_database::crate_details::{Release, parse_doc_targets}; use docs_rs_types::{BuildStatus, CrateId, KrateName, ReqVersion, Version, VersionReq}; use docs_rs_utils::rustc_version::parse_rustc_date; use error::AxumNope; use page::TemplateData; use sentry::integrations::tower as sentry_tower; use serde::Serialize; -use serde_json::Value; use std::{ borrow::Cow, net::{IpAddr, Ipv4Addr, SocketAddr}, @@ -91,10 +90,10 @@ pub(crate) struct MatchedRelease { pub req_version: ReqVersion, /// the matched release - pub release: crate_details::Release, + pub release: Release, /// all releases since we have them anyways and so we can pass them to CrateDetails - pub(crate) all_releases: Vec, + pub(crate) all_releases: Vec, } impl MatchedRelease { @@ -251,7 +250,7 @@ async fn match_version( // first load and parse all versions of this crate, // `releases_for_crate` is already sorted, newest version first. - let releases = crate_details::releases_for_crate(conn, crate_id) + let releases = docs_rs_database::crate_details::releases_for_crate(conn, crate_id) .await .context("error fetching releases for crate")?; @@ -589,7 +588,7 @@ impl MetaData { target_name: row.target_name, rustdoc_status: row.rustdoc_status, default_target: row.default_target, - doc_targets: row.doc_targets.map(MetaData::parse_doc_targets), + doc_targets: row.doc_targets.map(parse_doc_targets), yanked: row.yanked, rustdoc_css_file: row .rustc_version @@ -598,12 +597,6 @@ impl MetaData { .transpose()?, }) } - - fn parse_doc_targets(targets: Value) -> Vec { - let mut targets: Vec = serde_json::from_value(targets).unwrap_or_default(); - targets.sort_unstable(); - targets - } } #[derive(Template)]