Respond to Matt's additional comments.

Author: arik-so

.github/workflows/build.yml

@@ -141,7 +141,7 @@ jobs:
         run: |
           cargo test --verbose --color always  -p lightning
           cargo test --verbose --color always  -p lightning-invoice
-          cargo test --verbose --color always -p lightning-graph-sync
+          cargo test --verbose --color always  -p lightning-rapid-gossip-sync
           cargo build --verbose  --color always -p lightning-persister
           cargo build --verbose  --color always -p lightning-background-processor
       - name: Test C Bindings Modifications on Rust ${{ matrix.toolchain }}
@@ -232,9 +232,9 @@ jobs:
           EXPECTED_ROUTING_GRAPH_SNAPSHOT_SHASUM: 05a5361278f68ee2afd086cc04a1f927a63924be451f3221d380533acfacc303
       - name: Fetch rapid graph sync reference input
         run: |
-          curl --verbose -L -o lightning-graph-sync/res/full_graph.lngossip https://bitcoin.ninja/ldk-compressed_graph-bc08df7542-2022-05-05.bin
-          echo "Sha sum: $(sha256sum lightning-graph-sync/res/full_graph.lngossip | awk '{ print $1 }')"
-          if [ "$(sha256sum lightning-graph-sync/res/full_graph.lngossip | awk '{ print $1 }')" != "${EXPECTED_RAPID_GOSSIP_SHASUM}" ]; then
+          curl --verbose -L -o lightning-rapid-gossip-sync/res/full_graph.lngossip https://bitcoin.ninja/ldk-compressed_graph-bc08df7542-2022-05-05.bin
+          echo "Sha sum: $(sha256sum lightning-rapid-gossip-sync/res/full_graph.lngossip | awk '{ print $1 }')"
+          if [ "$(sha256sum lightning-rapid-gossip-sync/res/full_graph.lngossip | awk '{ print $1 }')" != "${EXPECTED_RAPID_GOSSIP_SHASUM}" ]; then
             echo "Bad hash"
             exit 1
           fi

fuzz/src/process_network_graph.rs

@@ -5,7 +5,7 @@ use utils::test_logger;
 fn do_test(data: &[u8]) {
 	let block_hash = bitcoin::BlockHash::default();
 	let network_graph = lightning::routing::network_graph::NetworkGraph::new(block_hash);
-	lightning_graph_sync::processing::update_network_graph(&network_graph, data);
+	lightning_rapid_gossip_sync::processing::update_network_graph(&network_graph, data);
 }
 
 /// Method that needs to be added manually, {name}_test

lightning-rapid-gossip-sync/Cargo.toml

@@ -14,7 +14,7 @@ _bench_unstable = []
 
 [dependencies]
 lightning = { version = "0.0.106", path = "../lightning" }
-bitcoin = { version = "0.28.1", default-features = false, features = ["secp-recovery"] }
+bitcoin = { version = "0.28.1", default-features = false }
 
 [dev-dependencies]
 lightning = { version = "0.0.106", path = "../lightning", features = ["_test_utils"] }

lightning-rapid-gossip-sync/README.md

@@ -39,8 +39,8 @@ You will also notice that `NodeAnnouncement` messages are omitted altogether as
 implicitly extracted from the channel announcements and updates.
 
 The data is then applied to the current network graph, artificially dated to the timestamp of the
-latest seen message, be it an announcement or an update, from the server's perspective. The network
-graph should not be pruned until the graph sync completes.
+latest seen message less one week, be it an announcement or an update, from the server's
+perspective. The network graph should not be pruned until the graph sync completes.
 
 ### Custom Channel Announcement
 
@@ -83,8 +83,8 @@ non-default or to have mutated.
 
 ## Delta Calculation
 
-The way a server is meant to calculate this rapid gossip sync data is by using the latest timestamp
-an change, be it either an announcement or an update, was seen. That timestamp is included in each
+The way a server is meant to calculate this rapid gossip sync data is by taking the latest time
+any change, be it either an announcement or an update, was seen. That timestamp is included in each
 rapid sync message, so all the client needs to do is cache one variable.
 
 If a particular channel update had never occurred before, the full update is sent. If a channel has
@@ -98,8 +98,8 @@ broadcast on the network may be taken into account when calculating the delta.
 ## Performance
 
 Given the primary purpose of this utility is a faster graph sync, we thought it might be helpful to
-provide some examples of various delta sets. These examples were calculated as of May 19th with a
-network graph comprised of 80,000 channel announcements and 160,000 directed channel updates.
+provide some examples of various delta sets. These examples were calculated as of May 19th  2022
+with a network graph comprised of 80,000 channel announcements and 160,000 directed channel updates.
 
 | Full sync                   |        |
 |-----------------------------|--------|

lightning-rapid-gossip-sync/src/lib.rs

@@ -10,7 +10,7 @@
 //! This crate exposes functionality to rapidly sync gossip data, aimed primarily at mobile
 //! devices.
 //!
-//! The server sends a compressed response containing gossip data. The gossip data is
+//! The server sends a compressed response containing differential gossip data. The gossip data is
 //! formatted compactly, omitting signatures and opportunistically incremental where previous
 //! channel updates are known (a mechanism that is enabled when the timestamp of the last known
 //! channel update is communicated). A reference server implementation can be found
@@ -21,23 +21,15 @@
 //! privately calculate routes for payments, and do so much faster and earlier than requiring a full
 //! peer-to-peer gossip sync to complete.
 //!
-//! The reason the rapid sync server requires a modicum of trust is that it could provide bogus
-//! data, though at worst, all that would result in is a fake network topology, which wouldn't
-//! enable the server to steal or siphon off funds. It could, however, reduce the client's privacy
-//! by increasing the likelihood payments will are routed via channels the server controls.
+//! The reason the rapid sync server requires trust is that it could provide bogus data, though at
+//! worst, all that would result in is a fake network topology, which wouldn't enable the server to
+//! steal or siphon off funds. It could, however, reduce the client's privacy by forcing all
+//! payments to be routed via channels the server controls.
 //!
 //! The way a server is meant to calculate this rapid gossip sync data is by using a `latest_seen`
 //! timestamp provided by the client. It's not included in either channel announcement or update,
 //! (not least due to announcements not including any timestamps at all, but only a block height)
 //! but rather, it's a timestamp of when the server saw a particular message.
-//!
-//! If a particular channel update had never occurred before, the full update is sent. If a channel
-//! has had updates prior to the provided timestamp, the latest update prior to the timestamp is
-//! taken as a reference, and the delta is calculated against it.
-//!
-//! In practice, snapshots of server data will be calculated at regular intervals, such that deltas
-//! are cached and needn't be calculated on the fly. All the client needs to provide to the server
-//! is the timestamp included in the previously received (and hence snapshotted) delta.
 
 // Allow and import test features for benching
 #![cfg_attr(all(test, feature = "_bench_unstable"), feature(test))]
@@ -65,7 +57,7 @@ pub mod processing;
 pub fn sync_network_graph_with_file_path(
 	network_graph: &network_graph::NetworkGraph,
 	sync_path: &str,
-) -> Result<(), GraphSyncError> {
+) -> Result<(u32), GraphSyncError> {
 	let mut file = File::open(sync_path)?;
 	processing::update_network_graph_from_byte_stream(&network_graph, &mut file)
 }

lightning-rapid-gossip-sync/src/processing.rs

@@ -31,15 +31,15 @@ const MAX_INITIAL_NODE_ID_VECTOR_CAPACITY: u32 = 50_000;
 pub fn update_network_graph(
 	network_graph: &network_graph::NetworkGraph,
 	update_data: &[u8],
-) -> Result<(), GraphSyncError> {
+) -> Result<(u32), GraphSyncError> {
 	let mut read_cursor = io::Cursor::new(update_data);
 	update_network_graph_from_byte_stream(&network_graph, &mut read_cursor)
 }
 
 pub(crate) fn update_network_graph_from_byte_stream<R: Read>(
 	network_graph: &network_graph::NetworkGraph,
 	mut read_cursor: &mut R,
-) -> Result<(), GraphSyncError> {
+) -> Result<(u32), GraphSyncError> {
 	let mut prefix = [0u8; 4];
 	read_cursor.read_exact(&mut prefix)?;
 
@@ -52,6 +52,8 @@ pub(crate) fn update_network_graph_from_byte_stream<R: Read>(
 
 	let chain_hash: BlockHash = Readable::read(read_cursor)?;
 	let latest_seen_timestamp: u32 = Readable::read(read_cursor)?;
+	// backdate the applied timestamp by a week
+	let backdated_timestamp = latest_seen_timestamp - 24 * 3600 * 7;
 
 	let node_id_count: u32 = Readable::read(read_cursor)?;
 	let mut node_ids: Vec<PublicKey> = Vec::with_capacity(std::cmp::min(
@@ -85,7 +87,7 @@ pub(crate) fn update_network_graph_from_byte_stream<R: Read>(
 
 		let announcement_result = network_graph.add_channel_from_partial_announcement(
 			short_channel_id,
-			latest_seen_timestamp as u64,
+			backdated_timestamp as u64,
 			features,
 			node_id_1,
 			node_id_2,
@@ -103,7 +105,7 @@ pub(crate) fn update_network_graph_from_byte_stream<R: Read>(
 
 	let update_count: u32 = Readable::read(read_cursor)?;
 	if update_count == 0 {
-		return Ok(());
+		return Ok((latest_seen_timestamp));
 	}
 
 	// obtain default values for non-incremental updates
@@ -135,7 +137,7 @@ pub(crate) fn update_network_graph_from_byte_stream<R: Read>(
 			UnsignedChannelUpdate {
 				chain_hash,
 				short_channel_id,
-				timestamp: latest_seen_timestamp,
+				timestamp: backdated_timestamp,
 				flags: standard_channel_flags,
 				cltv_expiry_delta: default_cltv_expiry_delta,
 				htlc_minimum_msat: default_htlc_minimum_msat,
@@ -172,7 +174,7 @@ pub(crate) fn update_network_graph_from_byte_stream<R: Read>(
 			UnsignedChannelUpdate {
 				chain_hash,
 				short_channel_id,
-				timestamp: latest_seen_timestamp,
+				timestamp: backdated_timestamp,
 				flags: standard_channel_flags,
 				cltv_expiry_delta: directional_info.cltv_expiry_delta,
 				htlc_minimum_msat: directional_info.htlc_minimum_msat,
@@ -216,7 +218,7 @@ pub(crate) fn update_network_graph_from_byte_stream<R: Read>(
 		network_graph.update_channel_unsigned(&synthetic_update)?;
 	}
 
-	Ok(())
+	Ok((latest_seen_timestamp))
 }
 
 #[cfg(test)]