Use download counts in search

Author: lpil

src/packages/override.gleam

@@ -45,3 +45,15 @@ pub fn expand_search_term(term: String) -> List(String) {
     term -> [term]
   }
 }
+
+pub fn is_core_package(name: String) -> Bool {
+  case name {
+    "gleam_stdlib"
+    | "gleam_javascript"
+    | "gleam_erlang"
+    | "gleam_otp"
+    | "gleam_json"
+    | "gleam_time" -> True
+    _ -> False
+  }
+}

src/packages/router.gleam

@@ -10,7 +10,6 @@ import gleam/time/calendar
 import gleam/time/timestamp
 import gleam/uri
 import packages/storage
-import packages/text_search
 import packages/web.{type Context}
 import packages/web/page
 import wisp.{type Request, type Response}
@@ -142,17 +141,8 @@ fn internet_points(ctx: Context) -> Response {
 fn search(request: Request, context: Context) -> Response {
   let search_term = get_search_parameter(request)
   let assert Ok(packages) = case search_term {
-    "" -> storage.list_packages(context.db)
-    _ -> text_search.lookup(context.search_index, search_term)
-  }
-  let assert Ok(packages) =
-    storage.ranked_package_summaries(context.db, packages, search_term)
-  let packages = case search_term {
-    "" ->
-      list.sort(packages, fn(a, b) {
-        timestamp.compare(b.updated_in_hex_at, a.updated_in_hex_at)
-      })
-    _ -> packages
+    "" -> storage.packages_most_recent_first(context.db)
+    _ -> storage.search_packages(context.db, context.search_index, search_term)
   }
 
   page.packages_list(packages, search_term)

src/packages/storage.gleam

@@ -1,4 +1,3 @@
-import gleam/bool
 import gleam/dict
 import gleam/dynamic/decode.{type Decoder}
 import gleam/float
@@ -14,6 +13,7 @@ import gleam/time/calendar
 import gleam/time/timestamp.{type Timestamp}
 import packages/error.{type Error}
 import packages/override
+import packages/text_search
 import storail.{type Collection}
 
 pub opaque type Database {
@@ -370,72 +370,63 @@ pub fn list_packages(database: Database) -> Result(List(String), Error) {
   }
 }
 
-type Groups {
-  Groups(
-    exact: List(Package),
-    regular: List(Package),
-    v0: List(Package),
-    old: List(Package),
-  )
-}
-
-pub fn ranked_package_summaries(
-  database: Database,
-  packages: List(String),
+pub fn search_packages(
+  db: Database,
+  search: text_search.TextSearchIndex,
   search_term: String,
 ) -> Result(List(Package), Error) {
-  let gleam_v1 =
-    timestamp.from_calendar(
-      calendar.Date(2024, calendar.March, 4),
-      calendar.TimeOfDay(0, 0, 0, 0),
-      calendar.utc_offset,
-    )
-
-  use packages <- result.map({
-    use name <- list.try_map(packages)
-    get_package(database, name)
-  })
+  let bool = fn(b) {
+    case b {
+      True -> 1
+      False -> 0
+    }
+  }
+  use found <- result.try(text_search.lookup(search, search_term))
+  use packages <- result.map(
+    list.try_map(found, fn(found) {
+      use package <- result.map(get_package(db, found.name))
+
+      let exact_package_name_match = bool(search_term == package.name)
+      let is_not_v0 = bool(!string.starts_with(package.latest_version, "0."))
+      let is_core_package = bool(override.is_core_package(package.name))
+      let updated_at =
+        float.round(timestamp.to_unix_seconds(package.updated_in_hex_at))
+
+      // This is the value we use to determine what order packages should be
+      // shown by. Later list values only take effect if the earlier ones are
+      // equal.
+      let ordering_key = [
+        exact_package_name_match,
+        is_not_v0,
+        found.match_count,
+        is_core_package,
+        package.downloads_recent,
+        updated_at,
+      ]
+      #(ordering_key, package)
+    }),
+  )
 
-  let groups = Groups([], [], [], [])
-
-  let groups =
-    list.fold(packages, groups, fn(groups, package) {
-      // The ordering of the clauses matter. Something can be both v0 and old,
-      // and which group it goes into impacts the final ordering.
-
-      use <- bool.lazy_guard(package.name == search_term, fn() {
-        Groups(..groups, exact: [package, ..groups.exact])
-      })
-
-      let is_old =
-        timestamp.compare(package.updated_in_hex_at, gleam_v1) == order.Lt
-      use <- bool.lazy_guard(is_old, fn() {
-        Groups(..groups, old: [package, ..groups.old])
-      })
-
-      let is_zero_version = string.starts_with(package.latest_version, "0.")
-      use <- bool.lazy_guard(is_zero_version, fn() {
-        Groups(..groups, v0: [package, ..groups.v0])
-      })
-
-      Groups(..groups, regular: [package, ..groups.regular])
-    })
-
-  let Groups(exact:, regular:, v0:, old:) = groups
-  // This list is ordered backwards, so the later in the list the higher it
-  // will be shown in the UI.
-  [
-    // Packages published before Gleam v1.0.0 are likely outdated.
-    old,
-    // v0 versions are discouraged, so they are shown lower.
-    v0,
-    // Regular versions are not prioritised in any particular way.
-    regular,
-    // Exact matches for the search term come first.
-    exact,
-  ]
-  |> list.flatten
-  |> list.reverse
+  packages
+  |> list.sort(fn(a, b) { list_compare(b.0, a.0, int.compare) })
+  |> list.map(fn(pair) { pair.1 })
+}
+
+fn list_compare(
+  a: List(t),
+  b: List(t),
+  compare: fn(t, t) -> order.Order,
+) -> order.Order {
+  case a, b {
+    [], [] -> order.Eq
+    [], _ -> order.Lt
+    _, [] -> order.Gt
+    [a1, ..a], [b1, ..b] ->
+      case compare(a1, b1) {
+        order.Eq -> list_compare(a, b, compare)
+        order.Gt as order | order.Lt as order -> order
+      }
+  }
 }
 
 pub fn try_fold_packages(
@@ -564,3 +555,11 @@ fn date_string(timestamp: Timestamp) -> String {
   |> timestamp.to_rfc3339(calendar.utc_offset)
   |> string.slice(0, 10)
 }
+
+pub fn packages_most_recent_first(db: Database) -> Result(List(Package), Error) {
+  use packages <- result.try(list_packages(db))
+  use packages <- result.map(list.try_map(packages, get_package(db, _)))
+  list.sort(packages, fn(a, b) {
+    timestamp.compare(b.updated_in_hex_at, a.updated_in_hex_at)
+  })
+}

src/packages/text_search.gleam

@@ -1,9 +1,7 @@
 import ethos.{type BagTable}
 import gleam/dict
-import gleam/int
 import gleam/list
 import gleam/option
-import gleam/order
 import gleam/result
 import gleam/string
 import packages/error.{type Error}
@@ -46,10 +44,12 @@ pub fn update(
   insert(index, name, description)
 }
 
+/// Find all matches for the given search term. The list is not returned in any
+/// order, but each found item is returned with a match count.
 pub fn lookup(
   index: TextSearchIndex,
   phrase: String,
-) -> Result(List(String), Error) {
+) -> Result(List(Found), Error) {
   let phrase = string.lowercase(phrase)
   stem_words(phrase)
   |> list.flat_map(override.expand_search_term)
@@ -61,32 +61,15 @@ pub fn lookup(
       dict.upsert(counters, name, fn(x) { option.unwrap(x, 0) + 1 })
     })
     |> dict.to_list
-    |> list.map(fn(pair) {
-      case pair.0 {
-        // Rank up proritised packages
-        "gleam_stdlib"
-        | "gleam_javascript"
-        | "gleam_erlang"
-        | "gleam_otp"
-        | "gleam_json"
-        | "gleam_time" -> #(pair.0, pair.1 + 10)
-        _ -> pair
-      }
-    })
-    |> list.sort(fn(a, b) {
-      case a, b {
-        // Exact matches come first
-        #(name, _), _ if name == phrase -> order.Lt
-        _, #(name, _) if name == phrase -> order.Gt
-        // Otherwise compare the score
-        _, _ -> int.compare(b.1, a.1)
-      }
-    })
-    |> list.map(fn(pair) { pair.0 })
+    |> list.map(fn(pair) { Found(pair.0, pair.1) })
   })
   |> result.replace_error(error.EtsTableError)
 }
 
+pub type Found {
+  Found(name: String, match_count: Int)
+}
+
 fn remove(index: TextSearchIndex, name: String) -> Result(Nil, Error) {
   ethos.delete_value(index.table, name)
   |> result.replace_error(error.EtsTableError)