Skip to content

Commit a58ae4c

Browse files
arik-soarik-so
committedMay 25, 2022
Introduce graph sync crate for fast-forwarding through gossip data downloaded from a server.
1 parent 75ca50f commit a58ae4c

19 files changed

+1185
-64
lines changed
 

‎.editorconfig

+1
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@
33
[*]
44
indent_style = tab
55
insert_final_newline = true
6+
trim_trailing_whitespace = true

‎.github/workflows/build.yml

+16-2
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ jobs:
141141
run: |
142142
cargo test --verbose --color always -p lightning
143143
cargo test --verbose --color always -p lightning-invoice
144+
cargo test --verbose --color always -p lightning-rapid-gossip-sync
144145
cargo build --verbose --color always -p lightning-persister
145146
cargo build --verbose --color always -p lightning-background-processor
146147
- name: Test C Bindings Modifications on Rust ${{ matrix.toolchain }}
@@ -221,11 +222,24 @@ jobs:
221222
- name: Fetch routing graph snapshot
222223
if: steps.cache-graph.outputs.cache-hit != 'true'
223224
run: |
224-
wget -O lightning/net_graph-2021-05-31.bin https://bitcoin.ninja/ldk-net_graph-v0.0.15-2021-05-31.bin
225-
if [ "$(sha256sum lightning/net_graph-2021-05-31.bin | awk '{ print $1 }')" != "05a5361278f68ee2afd086cc04a1f927a63924be451f3221d380533acfacc303" ]; then
225+
curl --verbose -L -o lightning/net_graph-2021-05-31.bin https://bitcoin.ninja/ldk-net_graph-v0.0.15-2021-05-31.bin
226+
echo "Sha sum: $(sha256sum lightning/net_graph-2021-05-31.bin | awk '{ print $1 }')"
227+
if [ "$(sha256sum lightning/net_graph-2021-05-31.bin | awk '{ print $1 }')" != "${EXPECTED_ROUTING_GRAPH_SNAPSHOT_SHASUM}" ]; then
226228
echo "Bad hash"
227229
exit 1
228230
fi
231+
env:
232+
EXPECTED_ROUTING_GRAPH_SNAPSHOT_SHASUM: 05a5361278f68ee2afd086cc04a1f927a63924be451f3221d380533acfacc303
233+
- name: Fetch rapid graph sync reference input
234+
run: |
235+
curl --verbose -L -o lightning-rapid-gossip-sync/res/full_graph.lngossip https://bitcoin.ninja/ldk-compressed_graph-bc08df7542-2022-05-05.bin
236+
echo "Sha sum: $(sha256sum lightning-rapid-gossip-sync/res/full_graph.lngossip | awk '{ print $1 }')"
237+
if [ "$(sha256sum lightning-rapid-gossip-sync/res/full_graph.lngossip | awk '{ print $1 }')" != "${EXPECTED_RAPID_GOSSIP_SHASUM}" ]; then
238+
echo "Bad hash"
239+
exit 1
240+
fi
241+
env:
242+
EXPECTED_RAPID_GOSSIP_SHASUM: 9637b91cea9d64320cf48fc0787c70fe69fc062f90d3512e207044110cadfd7b
229243
- name: Test with Network Graph on Rust ${{ matrix.toolchain }}
230244
run: |
231245
cd lightning

‎Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ members = [
77
"lightning-net-tokio",
88
"lightning-persister",
99
"lightning-background-processor",
10+
"lightning-rapid-gossip-sync"
1011
]
1112

1213
exclude = [

‎fuzz/Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ stdin_fuzz = []
1919
[dependencies]
2020
afl = { version = "0.4", optional = true }
2121
lightning = { path = "../lightning", features = ["regex"] }
22+
lightning-rapid-gossip-sync = { path = "../lightning-rapid-gossip-sync" }
2223
bitcoin = { version = "0.28.1", features = ["secp-lowmemory"] }
2324
hex = "0.3"
2425
honggfuzz = { version = "0.5", optional = true }

‎fuzz/src/bin/gen_target.sh

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ GEN_TEST chanmon_deser
1010
GEN_TEST chanmon_consistency
1111
GEN_TEST full_stack
1212
GEN_TEST peer_crypt
13+
GEN_TEST process_network_graph
1314
GEN_TEST router
1415
GEN_TEST zbase32
1516

‎fuzz/src/bin/process_network_graph_target.rs

+113
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// This file is Copyright its original authors, visible in version control
2+
// history.
3+
//
4+
// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
5+
// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6+
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
7+
// You may not use this file except in accordance with one or both of these
8+
// licenses.
9+
10+
// This file is auto-generated by gen_target.sh based on target_template.txt
11+
// To modify it, modify target_template.txt and run gen_target.sh instead.
12+
13+
#![cfg_attr(feature = "libfuzzer_fuzz", no_main)]
14+
15+
#[cfg(not(fuzzing))]
16+
compile_error!("Fuzz targets need cfg=fuzzing");
17+
18+
extern crate lightning_fuzz;
19+
use lightning_fuzz::process_network_graph::*;
20+
21+
#[cfg(feature = "afl")]
22+
#[macro_use] extern crate afl;
23+
#[cfg(feature = "afl")]
24+
fn main() {
25+
fuzz!(|data| {
26+
process_network_graph_run(data.as_ptr(), data.len());
27+
});
28+
}
29+
30+
#[cfg(feature = "honggfuzz")]
31+
#[macro_use] extern crate honggfuzz;
32+
#[cfg(feature = "honggfuzz")]
33+
fn main() {
34+
loop {
35+
fuzz!(|data| {
36+
process_network_graph_run(data.as_ptr(), data.len());
37+
});
38+
}
39+
}
40+
41+
#[cfg(feature = "libfuzzer_fuzz")]
42+
#[macro_use] extern crate libfuzzer_sys;
43+
#[cfg(feature = "libfuzzer_fuzz")]
44+
fuzz_target!(|data: &[u8]| {
45+
process_network_graph_run(data.as_ptr(), data.len());
46+
});
47+
48+
#[cfg(feature = "stdin_fuzz")]
49+
fn main() {
50+
use std::io::Read;
51+
52+
let mut data = Vec::with_capacity(8192);
53+
std::io::stdin().read_to_end(&mut data).unwrap();
54+
process_network_graph_run(data.as_ptr(), data.len());
55+
}
56+
57+
#[test]
58+
fn run_test_cases() {
59+
use std::fs;
60+
use std::io::Read;
61+
use lightning_fuzz::utils::test_logger::StringBuffer;
62+
63+
use std::sync::{atomic, Arc};
64+
{
65+
let data: Vec<u8> = vec![0];
66+
process_network_graph_run(data.as_ptr(), data.len());
67+
}
68+
let mut threads = Vec::new();
69+
let threads_running = Arc::new(atomic::AtomicUsize::new(0));
70+
if let Ok(tests) = fs::read_dir("test_cases/process_network_graph") {
71+
for test in tests {
72+
let mut data: Vec<u8> = Vec::new();
73+
let path = test.unwrap().path();
74+
fs::File::open(&path).unwrap().read_to_end(&mut data).unwrap();
75+
threads_running.fetch_add(1, atomic::Ordering::AcqRel);
76+
77+
let thread_count_ref = Arc::clone(&threads_running);
78+
let main_thread_ref = std::thread::current();
79+
threads.push((path.file_name().unwrap().to_str().unwrap().to_string(),
80+
std::thread::spawn(move || {
81+
let string_logger = StringBuffer::new();
82+
83+
let panic_logger = string_logger.clone();
84+
let res = if ::std::panic::catch_unwind(move || {
85+
process_network_graph_test(&data, panic_logger);
86+
}).is_err() {
87+
Some(string_logger.into_string())
88+
} else { None };
89+
thread_count_ref.fetch_sub(1, atomic::Ordering::AcqRel);
90+
main_thread_ref.unpark();
91+
res
92+
})
93+
));
94+
while threads_running.load(atomic::Ordering::Acquire) > 32 {
95+
std::thread::park();
96+
}
97+
}
98+
}
99+
let mut failed_outputs = Vec::new();
100+
for (test, thread) in threads.drain(..) {
101+
if let Some(output) = thread.join().unwrap() {
102+
println!("\nOutput of {}:\n{}\n", test, output);
103+
failed_outputs.push(test);
104+
}
105+
}
106+
if !failed_outputs.is_empty() {
107+
println!("Test cases which failed: ");
108+
for case in failed_outputs {
109+
println!("{}", case);
110+
}
111+
panic!();
112+
}
113+
}

‎fuzz/src/lib.rs

+2
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
extern crate bitcoin;
1111
extern crate lightning;
12+
extern crate lightning_rapid_gossip_sync;
1213
extern crate hex;
1314

1415
pub mod utils;
@@ -17,6 +18,7 @@ pub mod chanmon_deser;
1718
pub mod chanmon_consistency;
1819
pub mod full_stack;
1920
pub mod peer_crypt;
21+
pub mod process_network_graph;
2022
pub mod router;
2123
pub mod zbase32;
2224

‎fuzz/src/process_network_graph.rs

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Import that needs to be added manually
2+
use utils::test_logger;
3+
4+
/// Actual fuzz test, method signature and name are fixed
5+
fn do_test(data: &[u8]) {
6+
let block_hash = bitcoin::BlockHash::default();
7+
let network_graph = lightning::routing::network_graph::NetworkGraph::new(block_hash);
8+
lightning_rapid_gossip_sync::processing::update_network_graph(&network_graph, data);
9+
}
10+
11+
/// Method that needs to be added manually, {name}_test
12+
pub fn process_network_graph_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
13+
do_test(data);
14+
}
15+
16+
/// Method that needs to be added manually, {name}_run
17+
#[no_mangle]
18+
pub extern "C" fn process_network_graph_run(data: *const u8, datalen: usize) {
19+
do_test(unsafe { std::slice::from_raw_parts(data, datalen) });
20+
}

‎fuzz/targets.h

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ void chanmon_deser_run(const unsigned char* data, size_t data_len);
33
void chanmon_consistency_run(const unsigned char* data, size_t data_len);
44
void full_stack_run(const unsigned char* data, size_t data_len);
55
void peer_crypt_run(const unsigned char* data, size_t data_len);
6+
void process_network_graph_run(const unsigned char* data, size_t data_len);
67
void router_run(const unsigned char* data, size_t data_len);
78
void zbase32_run(const unsigned char* data, size_t data_len);
89
void msg_accept_channel_run(const unsigned char* data, size_t data_len);

‎lightning-rapid-gossip-sync/Cargo.toml

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
[package]
2+
name = "lightning-rapid-gossip-sync"
3+
version = "0.0.104"
4+
authors = ["Arik Sosman <git@arik.io>"]
5+
license = "MIT OR Apache-2.0"
6+
repository = "https://github.com/lightningdevkit/rust-lightning"
7+
edition = "2018"
8+
description = """
9+
Utility to process gossip routing data from Rapid Gossip Sync Server.
10+
"""
11+
12+
[features]
13+
_bench_unstable = []
14+
15+
[dependencies]
16+
lightning = { version = "0.0.106", path = "../lightning" }
17+
bitcoin = { version = "0.28.1", default-features = false }
18+
19+
[dev-dependencies]
20+
lightning = { version = "0.0.106", path = "../lightning", features = ["_test_utils"] }

‎lightning-rapid-gossip-sync/README.md

+120
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
# lightning-rapid-gossip-sync
2+
3+
This crate exposes functionality for rapid gossip graph syncing, aimed primarily at mobile clients.
4+
Its server counterpart is the
5+
[rapid-gossip-sync-server](https://github.com/lightningdevkit/rapid-gossip-sync-server) repository.
6+
7+
## Mechanism
8+
9+
The (presumed) server sends a compressed gossip response containing gossip data. The gossip data is
10+
formatted compactly, omitting signatures and opportunistically incremental where previous channel
11+
updates are known.
12+
13+
Essentially, the serialization structure is as follows:
14+
15+
1. Fixed prefix bytes `76, 68, 75, 1` (the first three bytes are ASCII for `LDK`)
16+
- The purpose of this prefix is to identify the serialization format, should other rapid gossip
17+
sync formats arise in the future
18+
- The fourth byte is the protocol version in case our format gets updated
19+
2. Chain hash (32 bytes)
20+
3. Latest seen timestamp (`u32`)
21+
4. An unsigned int indicating the number of node IDs to follow
22+
5. An array of compressed node ID pubkeys (all pubkeys are presumed to be standard
23+
compressed 33-byte-serializations)
24+
6. An unsigned int indicating the number of channel announcement messages to follow
25+
7. An array of significantly stripped down customized channel announcements
26+
8. An unsigned int indicating the number of channel update messages to follow
27+
9. A series of default values used for non-incremental channel updates
28+
- The values are defined as follows:
29+
1. `default_cltv_expiry_delta`
30+
2. `default_htlc_minimum_msat`
31+
3. `default_fee_base_msat`
32+
4. `default_fee_proportional_millionths`
33+
5. `default_htlc_maximum_msat` (`u64`, and if the default is no maximum, `u64::MAX`)
34+
- The defaults are calculated by the server based on the frequency among non-incremental
35+
updates within a given delta set
36+
10. An array of customized channel updates
37+
38+
You will also notice that `NodeAnnouncement` messages are omitted altogether as the node IDs are
39+
implicitly extracted from the channel announcements and updates.
40+
41+
The data is then applied to the current network graph, artificially dated to the timestamp of the
42+
latest seen message less one week, be it an announcement or an update, from the server's
43+
perspective. The network graph should not be pruned until the graph sync completes.
44+
45+
### Custom Channel Announcement
46+
47+
To achieve compactness and avoid data repetition, we're sending a significantly stripped down
48+
version of the channel announcement message, which contains only the following data:
49+
50+
1. `channel_features`: `u16` + `n`, where `n` is the number of bytes indicated by the first `u16`
51+
2. `short_channel_id`: `CompactSize` (incremental `CompactSize` deltas starting from 0)
52+
3. `node_id_1_index`: `CompactSize` (index of node id within the previously sent sequence)
53+
4. `node_id_2_index`: `CompactSize` (index of node id within the previously sent sequence)
54+
55+
### Custom Channel Update
56+
57+
For the purpose of rapid syncing, we have deviated from the channel update format specified in
58+
BOLT 7 significantly. Our custom channel updates are structured as follows:
59+
60+
1. `short_channel_id`: `CompactSize` (incremental `CompactSize` deltas starting at 0)
61+
2. `custom_channel_flags`: `u8`
62+
3. `update_data`
63+
64+
Specifically, our custom channel flags break down like this:
65+
66+
| 128 | 64 | 32 | 16 | 8 | 4 | 2 | 1 |
67+
|---------------------|----|----|----|---|---|------------------|-----------|
68+
| Incremental update? | | | | | | Disable channel? | Direction |
69+
70+
If the most significant bit is set to `1`, indicating an incremental update, the intermediate bit
71+
flags assume the following meaning:
72+
73+
| 64 | 32 | 16 | 8 | 4 |
74+
|---------------------------------|---------------------------------|-----------------------------|-------------------------------------------|---------------------------------|
75+
| `cltv_expiry_delta` has changed | `htlc_minimum_msat` has changed | `fee_base_msat` has changed | `fee_proportional_millionths` has changed | `htlc_maximum_msat` has changed |
76+
77+
If the most significant bit is set to `0`, the meaning is almost identical, except instead of a
78+
change, the flags now represent a deviation from the defaults sent at the beginning of the update
79+
sequence.
80+
81+
In both cases, `update_data` only contains the fields that are indicated by the channel flags to be
82+
non-default or to have mutated.
83+
84+
## Delta Calculation
85+
86+
The way a server is meant to calculate this rapid gossip sync data is by taking the latest time
87+
any change, be it either an announcement or an update, was seen. That timestamp is included in each
88+
rapid sync message, so all the client needs to do is cache one variable.
89+
90+
If a particular channel update had never occurred before, the full update is sent. If a channel has
91+
had updates prior to the provided timestamp, the latest update prior to the timestamp is taken as a
92+
reference, and the delta is calculated against it.
93+
94+
Depending on whether the rapid sync message is calculated on the fly or a snapshotted version is
95+
returned, intermediate changes between the latest update seen by the client and the latest update
96+
broadcast on the network may be taken into account when calculating the delta.
97+
98+
## Performance
99+
100+
Given the primary purpose of this utility is a faster graph sync, we thought it might be helpful to
101+
provide some examples of various delta sets. These examples were calculated as of May 19th 2022
102+
with a network graph comprised of 80,000 channel announcements and 160,000 directed channel updates.
103+
104+
| Full sync | |
105+
|-----------------------------|--------|
106+
| Message Length | 4.7 MB |
107+
| Gzipped Message Length | 2.0 MB |
108+
| Client-side Processing Time | 1.4 s |
109+
110+
| Week-old sync | |
111+
|-----------------------------|--------|
112+
| Message Length | 2.7 MB |
113+
| Gzipped Message Length | 862 kB |
114+
| Client-side Processing Time | 907 ms |
115+
116+
| Day-old sync | |
117+
|-----------------------------|---------|
118+
| Message Length | 191 kB |
119+
| Gzipped Message Length | 92.8 kB |
120+
| Client-side Processing Time | 196 ms |

‎lightning-rapid-gossip-sync/res/.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*
2+
!.gitignore

0 commit comments

Comments
 (0)
Please sign in to comment.