Resync all data periodically

Author: lpil

src/packages.gleam

@@ -23,6 +23,7 @@ import wisp/wisp_mist
 
 const usage = "Usage:
   gleam run server
+  gleam run sync
   gleam run sync --name PACKAGE_NAME
 "
 

src/packages/storage.gleam

@@ -17,7 +17,7 @@ import storail.{type Collection}
 
 pub opaque type Database {
   Database(
-    most_recent_hex_timestamp: Collection(Timestamp),
+    hex_sync_times: Collection(Timestamp),
     packages: Collection(Package),
     releases: Collection(Release),
   )
@@ -26,9 +26,9 @@ pub opaque type Database {
 pub fn initialise(storage_path: String) -> Database {
   let config = storail.Config(storage_path:)
 
-  let most_recent_hex_timestamp =
+  let hex_sync_times =
     storail.Collection(
-      name: "most_recent_hex_timestamp",
+      name: "hex_sync_times",
       to_json: json_timestamp,
       decoder: decode.int |> decode.map(timestamp.from_unix_seconds),
       config:,
@@ -50,7 +50,7 @@ pub fn initialise(storage_path: String) -> Database {
       config:,
     )
 
-  Database(most_recent_hex_timestamp:, packages:, releases:)
+  Database(hex_sync_times:, packages:, releases:)
 }
 
 const ignored_packages = [
@@ -210,24 +210,39 @@ fn release_decoder() -> Decoder(Release) {
   ))
 }
 
-/// Insert or replace the most recent Hex timestamp in the database.
-pub fn upsert_most_recent_hex_timestamp(
+pub type WhichHexSyncTime {
+  PartialSync
+  FullSync
+}
+
+fn which_hex_sync_time_key(which: WhichHexSyncTime) -> String {
+  case which {
+    PartialSync -> "most-recent-partial-sync"
+    FullSync -> "most-recent-full-sync"
+  }
+}
+
+/// Insert or replace the a Hex timestamp in the database.
+pub fn upsert_hex_sync_time(
   database: Database,
+  which: WhichHexSyncTime,
   time: Timestamp,
 ) -> Result(Nil, Error) {
-  database.most_recent_hex_timestamp
-  |> storail.key("latest")
+  database.hex_sync_times
+  |> storail.key(which_hex_sync_time_key(which))
   |> storail.write(time)
   |> result.map_error(error.StorageError)
 }
 
-/// Get the most recent Hex timestamp from the database, returning the Unix
-/// epoch if there is no previous timestamp in the database.
-pub fn get_most_recent_hex_timestamp(
+/// Get a Hex timestamp from the database, returning a time
+/// before the first package publication if there is no previous timestamp in
+/// the database.
+pub fn get_hex_sync_time(
   database: Database,
+  which: WhichHexSyncTime,
 ) -> Result(Timestamp, Error) {
-  database.most_recent_hex_timestamp
-  |> storail.key("latest")
+  database.hex_sync_times
+  |> storail.key(which_hex_sync_time_key(which))
   |> storail.optional_read
   |> result.map(option.unwrap(_, gleam_package_epoch()))
   |> result.map_error(error.StorageError)

src/packages/syncing.gleam

@@ -14,7 +14,7 @@ import gleam/time/duration
 import gleam/time/timestamp.{type Timestamp}
 import gleam/uri
 import packages/error.{type Error}
-import packages/storage.{type Database}
+import packages/storage.{type Database, FullSync, PartialSync}
 import packages/text_search
 import wisp
 
@@ -42,8 +42,26 @@ pub fn sync_new_gleam_releases(
   db: Database,
   text_search: text_search.TextSearchIndex,
 ) -> Result(Nil, Error) {
-  wisp.log_info("Syncing new releases from Hex")
-  use limit <- try(storage.get_most_recent_hex_timestamp(db))
+  let now = timestamp.system_time()
+  use latest_partial <- try(storage.get_hex_sync_time(db, PartialSync))
+  use latest_full <- try(storage.get_hex_sync_time(db, FullSync))
+
+  // Periodically a full-sync is performed, to get the most up-to-date
+  // information for all packages and releases instead of just those that have
+  // been published since the last sync.
+  let next_full_sync_deadline = timestamp.add(latest_full, duration.hours(12))
+  let #(limit, mode) = case timestamp.compare(now, next_full_sync_deadline) {
+    order.Gt | order.Eq -> {
+      wisp.log_info("Performing full Hex sync, getting all data")
+      #(latest_full, FullSync)
+    }
+    order.Lt -> {
+      wisp.log_info("Performing partial Hex sync, getting newly published data")
+      #(latest_partial, PartialSync)
+    }
+  }
+
+  // Perform the sync with the Hex API
   use latest <- try(
     sync_packages(State(
       page: 1,
@@ -55,9 +73,16 @@ pub fn sync_new_gleam_releases(
       text_search:,
     )),
   )
-  let latest = storage.upsert_most_recent_hex_timestamp(db, latest)
+
+  // Record stats, so we know how to scan next time.
+  use _ <- result.try(storage.upsert_hex_sync_time(db, PartialSync, latest))
+  use _ <- result.try(case mode {
+    FullSync -> storage.upsert_hex_sync_time(db, FullSync, latest)
+    PartialSync -> Ok(Nil)
+  })
+
   wisp.log_info("Up to date!")
-  latest
+  Ok(Nil)
 }
 
 pub fn fetch_and_sync_package(

test/packages/storage_test.gleam

@@ -4,23 +4,42 @@ import gleam/option.{None, Some}
 import gleam/time/calendar
 import gleam/time/timestamp
 import packages/error
-import packages/storage.{Package, Release}
+import packages/storage.{FullSync, Package, PartialSync, Release}
 import storail
 import tests
 
-pub fn most_recent_hex_timestamp_test() {
+pub fn partial_sync_time_test() {
   use db <- tests.with_database
 
-  let assert Ok(Nil) =
-    storage.upsert_most_recent_hex_timestamp(db, timestamp.from_unix_seconds(0))
-  let assert Ok(time) = storage.get_most_recent_hex_timestamp(db)
+  let timestamp = timestamp.from_unix_seconds(0)
+  let assert Ok(Nil) = storage.upsert_hex_sync_time(db, PartialSync, timestamp)
+  let assert Ok(time) = storage.get_hex_sync_time(db, PartialSync)
   let assert #(0, 0) = timestamp.to_unix_seconds_and_nanoseconds(time)
   let assert "1970-01-01T00:00:00Z" =
     timestamp.to_rfc3339(time, calendar.utc_offset)
 
   let timestamp = timestamp.from_unix_seconds(2_284_352_323)
-  let assert Ok(Nil) = storage.upsert_most_recent_hex_timestamp(db, timestamp)
-  let assert Ok(time) = storage.get_most_recent_hex_timestamp(db)
+  let assert Ok(Nil) = storage.upsert_hex_sync_time(db, PartialSync, timestamp)
+  let assert Ok(time) = storage.get_hex_sync_time(db, PartialSync)
+  let assert "2042-05-22T06:18:43Z" =
+    timestamp.to_rfc3339(time, calendar.utc_offset)
+  let assert #(2_284_352_323, 0) =
+    timestamp.to_unix_seconds_and_nanoseconds(time)
+}
+
+pub fn full_sync_time_test() {
+  use db <- tests.with_database
+
+  let timestamp = timestamp.from_unix_seconds(0)
+  let assert Ok(Nil) = storage.upsert_hex_sync_time(db, FullSync, timestamp)
+  let assert Ok(time) = storage.get_hex_sync_time(db, FullSync)
+  let assert #(0, 0) = timestamp.to_unix_seconds_and_nanoseconds(time)
+  let assert "1970-01-01T00:00:00Z" =
+    timestamp.to_rfc3339(time, calendar.utc_offset)
+
+  let timestamp = timestamp.from_unix_seconds(2_284_352_323)
+  let assert Ok(Nil) = storage.upsert_hex_sync_time(db, FullSync, timestamp)
+  let assert Ok(time) = storage.get_hex_sync_time(db, FullSync)
   let assert "2042-05-22T06:18:43Z" =
     timestamp.to_rfc3339(time, calendar.utc_offset)
   let assert #(2_284_352_323, 0) =