Merge pull request #488 from network-intelligence/new-cbor-fps

davidmcgrew · GitHub Enterprise · commit d3cd14b8b514 · 2025-08-08T12:26:26.000-04:00
Adding CBOR encoding and decoding for STUN fingerprints
diff --git a/doc/CHANGELOG.md b/doc/CHANGELOG.md
@@ -1,5 +1,6 @@
 # CHANGELOG for Mercury
 
+* Added CBOR encoding/decoding for SSH and STUN fingerprints.
 * Fixing compiler warnings related to ABI differences
 * CMake changes required to add xsimd as submodule and fixing
   windows compilation issues
diff --git a/doc/fdc.md b/doc/fdc.md
@@ -0,0 +1,39 @@
+# Fingerprint and Destination Context (FDC) Schema
+
+
+
+This note documents the data formats used in the Mercury Fingerprint and Destination Context (FDC).  The FDC is encoded using the Concise Binary Object Representation ([CBOR](https://datatracker.ietf.org/doc/html/rfc8949)), an IETF standard data format that is extensible, consice, and trivially mappable to the common JavaScript Object Notation ([JSON](https://datatracker.ietf.org/doc/html/rfc8259)).  The formats are formally defined using the Concise Data Definition Language ([CDDL](https://datatracker.ietf.org/doc/html/rfc8610)), an IETF standard notational convention for unamiguously expressing CBOR and JSON data formats. 
+
+A Fingerprint and Destination Context (FDC) object contains a Network Protocol Fingerprint ([NPF](https://github.com/cisco/mercury/blob/main/doc/npf.md)) and other data features, all of which are metadata observed in a single network session.  An NPF fingerprint is a set of data features formed by selecting and normalizing some elements of a protocol message, so that they are correlated with the sending application or library implementation.  A fingerprint by itself sometimes uniquely identifies an application, but often does not.   In the latter case, the other data features are valuable for indentifying the sending application.
+
+- An NPF fingerprint in CBOR encoding, as defined in the [NPF CDDL specification](https://github.com/cisco/mercury/blob/main/doc/npf.cddl). 
+- The server name, which corresponds to the TLS or QUIC Server Name field or the HTTP Host field.
+- The destination IP address, as a string containing a textual representation.
+- The destination port number, as an unsigned integer less than 64,535.
+- The user agent as a string, which corresponds to the value of the User-Agent header for HTTP, the value of the SOFTWARE attribute for STUN, and the concatenation of the Protocol and Comment strings for SSH.
+- Optionally, an unsigned integer corresponding to the truncation code, which indicates whether reassembly was required in order to obtain a complete fingerprint, and whether or not the fingerprint was truncated due to a missing packet.  Its values are 
+  - none = 0,
+  - reassembled = 1
+  - truncated = 2
+  - reassembled and truncated = 3.
+
+The protocol (TLS, QUIC, HTTP, STUN, SSH) is identified by the fingerprint.
+
+The formal CDDL defintion is as follows:
+
+```
+; fdc is a record-style array of data elements comprised of a
+; fingerprint and the associated destination context.
+;
+fdc = [
+   fingerprint,              ; as defined in npf.cddl
+   str,                      ; server name
+   str,                      ; destination IP address, textual representation
+   uint,                     ; destination port (max: 0xffff)
+   str,                      ; user agent
+   ? uint                    ; truncation
+]
+```
+
+​	
+
diff --git a/src/cbor.cpp b/src/cbor.cpp
@@ -29,6 +29,7 @@ int main(int argc, char *argv[]) {
             { argument::none,     "--encode-fingerprint", "encode fingerprint string as CBOR" },
             { argument::required, "--input-file",         "read data from file <filename>" },
             { argument::none,     "--verbose-tests",      "run unit tests in verbose mode" },
+            { argument::none,     "--verbose",            "provide verbose output" },
             { argument::none,     "--help",               "print out help message" },
         });
 
@@ -42,12 +43,15 @@ int main(int argc, char *argv[]) {
     bool decode_fdc     = opt.is_set("--decode-fdc");
     bool encode_fp      = opt.is_set("--encode-fingerprint");
     bool verbose_tests  = opt.is_set("--verbose-tests");
+    bool verbose        = opt.is_set("--verbose");
     bool help_needed    = opt.is_set("--help");
     if (help_needed) {
         opt.usage(stdout, argv[0], summary);
         return 0;
     }
 
+    FILE *verbose_output = verbose ? stderr : nullptr;
+
     if (verbose_tests) {
         bool sd_result = static_dictionary<0>::unit_test(stdout);
         bool cbor_result = cbor::unit_test(stdout);
@@ -105,7 +109,7 @@ int main(int argc, char *argv[]) {
                 fprintf(stdout, "\"dst_ip_str\": \"%s\",", dst_ip_str);
                 fprintf(stdout, "\"dst_port\": %u,", dst_port);
                 fprintf(stdout, "\"user-agent\": \"%s\",", ua_str);
-                fprintf(stdout, "\"truncation\": %u", truncation);
+                fprintf(stdout, "\"truncation\": %lu", truncation);
                 fprintf(stdout, "}}\n");
             } else {
                 fprintf(stderr, "error: could not decode FDC\n");
@@ -114,6 +118,8 @@ int main(int argc, char *argv[]) {
 
     } else if (encode_fp) {
 
+        size_t num_fails = 0;
+
         std::ios::sync_with_stdio(false);  // for performance
         std::string line;
         while (std::getline(std::cin, line)) {
@@ -123,9 +129,9 @@ int main(int argc, char *argv[]) {
 
             // verify that we can represent this fingerprint in CBOR
             //
-            if (!cbor_fingerprint::test_fingerprint(line.c_str())){
+            if (!cbor_fingerprint::test_fingerprint(line.c_str(), verbose_output)) {
                 fprintf(stderr, "error: could not encode/decode fingerprint %s\n", line.c_str());
-                return EXIT_FAILURE;
+                num_fails++;
             }
 
             // convert fingerprint to CBOR
@@ -142,6 +148,10 @@ int main(int argc, char *argv[]) {
             //
             cbor::decode_fprint(data_buf.contents(), stdout);
         }
+        if (num_fails) {
+            fprintf(stderr, "error: could not encode/decode %zu fingerprints\n", num_fails);
+            return EXIT_FAILURE;
+        }
         return EXIT_SUCCESS;
 
     } else {
diff --git a/src/libmerc/cbor.hpp b/src/libmerc/cbor.hpp
@@ -348,6 +348,12 @@ namespace cbor {
             buf << value__;
         }
 
+        /// writes an empty `byte_string` into \param buf
+        ///
+        static void write_empty(writeable &buf) {
+            uint64{0, byte_string_type}.write(buf);
+        }
+
         /// `cbor::byte_string::unit_test()` performs unit tests on
         /// the class \ref cbor::byte_string and returns `true` if
         /// they all pass, and `false` otherwise.  If \param f ==
diff --git a/src/libmerc/fdc.hpp b/src/libmerc/fdc.hpp
@@ -57,8 +57,13 @@ namespace cbor_fingerprint {
 
     inline void encode_cbor_data(datum &d, writeable &w) {
         literal_byte<'('>{d};
-        cbor::byte_string_from_hex{hex_digits{d}}.write(w);
-        literal_byte<')'>{d};
+        if (lookahead<literal_byte<')'>> close{d}) {
+            cbor::byte_string::write_empty(w);
+            d = close.advance();
+        } else {
+            cbor::byte_string_from_hex{hex_digits{d}}.write(w);
+            literal_byte<')'>{d};
+        }
     }
 
     inline void encode_cbor_list(datum &d, writeable &w) {
@@ -190,6 +195,55 @@ namespace cbor_fingerprint {
         m.close();
     }
 
+    inline void encode_cbor_stun_fingerprint(datum d, writeable &w) {
+        cbor::output::map m{w};
+
+        if (lookahead<literal_byte<'1', '/'>> version_one{d}) {
+            d = version_one.advance();
+            cbor::uint64{1}.write(w);      // fingerprint version
+
+            if (lookahead<literal_byte<'r', 'a', 'n', 'd', 'o', 'm', 'i', 'z', 'e', 'd'>> peek{d}) {
+                constexpr size_t idx = fp_labels.index("randomized");
+                cbor::uint64{idx}.write(m);
+            } else {
+                cbor::output::array a{w};
+                encode_cbor_data(d, a);         // class
+                encode_cbor_data(d, a);         // method
+                encode_cbor_data(d, a);         // magic
+                encode_cbor_list(d, a);         // attributes
+                a.close();
+            }
+
+        }
+        m.close();
+     }
+
+    inline void encode_cbor_ssh_fingerprint(datum d, writeable &w) {
+        cbor::output::map m{w};
+
+        cbor::uint64{0}.write(w);      // fingerprint version
+
+        if (lookahead<literal_byte<'r', 'a', 'n', 'd', 'o', 'm', 'i', 'z', 'e', 'd'>> peek{d}) {
+            constexpr size_t idx = fp_labels.index("randomized");
+            cbor::uint64{idx}.write(m);
+        } else {
+            cbor::output::array a{w};
+            encode_cbor_data(d, a);         // kex_algorithms
+            encode_cbor_data(d, a);         // server_host_key_algorithms
+            encode_cbor_data(d, a);         // encryption_algorithms_client_to_server
+            encode_cbor_data(d, a);         // encryption_algorithms_server_to_client
+            encode_cbor_data(d, a);         // mac_algorithms_client_to_server
+            encode_cbor_data(d, a);         // mac_algorithms_server_to_client
+            encode_cbor_data(d, a);         // compression_algorithms_client_to_server
+            encode_cbor_data(d, a);         // compression_algorithms_server_to_client
+            encode_cbor_data(d, a);         // languages_client_to_server
+            encode_cbor_data(d, a);         // languages_server_to_client
+            a.close();
+        }
+
+        m.close();
+     }
+
     constexpr uint64_t randomized = 0;
     constexpr uint64_t generic = 1;
 
@@ -255,13 +309,28 @@ namespace cbor_fingerprint {
             encode_cbor_tofsee_fingerprint(d, w);
             m.close();
 
+        } else if (lookahead<literal_byte<'s', 't', 'u', 'n', '/'>> stun{d}) {
+            fp_type = fingerprint_type_stun;
+            cbor::output::map m{w};
+            cbor::uint64{(uint64_t)fp_type}.write(w);
+            d = stun.advance();
+            encode_cbor_stun_fingerprint(d, w);
+            m.close();
+
+        } else if (lookahead<literal_byte<'s', 's', 'h', '/'>> ssh{d}) {
+            fp_type = fingerprint_type_ssh;
+            cbor::output::map m{w};
+            cbor::uint64{(uint64_t)fp_type}.write(w);
+            d = ssh.advance();
+            encode_cbor_ssh_fingerprint(d, w);
+            m.close();
+
         }
         // fprintf(stderr, "fingerprint type %d\n", fp_type);
     }
 
     inline void decode_cbor_data(datum &d, writeable &w) {
         cbor::byte_string data = cbor::byte_string::decode(d);
-        //        if (d.is_null()) { return; }
         w.copy('(');
         w.write_hex(data.value().data, data.value().length());
         w.copy(')');
@@ -391,6 +460,54 @@ namespace cbor_fingerprint {
         m.close();
     }
 
+    inline void decode_stun_fp(datum &d, writeable &w) {
+        cbor::map m{d};
+        cbor::uint64 format_version{m.value()};
+        if (format_version.value() == 1) {
+            w.copy('1');
+            w.copy('/');
+            if (lookahead<cbor::uint64> label{m.value()}) {
+                if (label.value.value() == fp_labels.index("randomized")) {
+                    w << datum{"randomized"};
+                }
+            } else {
+                cbor::array a{m.value()};
+                decode_cbor_data(m.value(), w);         // class
+                decode_cbor_data(m.value(), w);         // method
+                decode_cbor_data(m.value(), w);         // magic
+                decode_cbor_list(m.value(), w);         // attributes
+                a.close();
+            }
+        }
+        m.close();
+    }
+
+    inline void decode_ssh_fp(datum &d, writeable &w) {
+        cbor::map m{d};
+        cbor::uint64 format_version{m.value()};
+        if (format_version.value() == 0) {
+            if (lookahead<cbor::uint64> label{m.value()}) {
+                if (label.value.value() == fp_labels.index("randomized")) {
+                    w << datum{"randomized"};
+                }
+            } else {
+                cbor::array a{m.value()};
+                decode_cbor_data(m.value(), w);   // kex_algorithms
+                decode_cbor_data(m.value(), w);   // server_host_key_algorithms
+                decode_cbor_data(m.value(), w);   // encryption_algorithms_client_to_server
+                decode_cbor_data(m.value(), w);   // encryption_algorithms_server_to_client
+                decode_cbor_data(m.value(), w);   // mac_algorithms_client_to_server
+                decode_cbor_data(m.value(), w);   // mac_algorithms_server_to_client
+                decode_cbor_data(m.value(), w);   // compression_algorithms_client_to_server
+                decode_cbor_data(m.value(), w);   // compression_algorithms_server_to_client
+                decode_cbor_data(m.value(), w);   // languages_client_to_server
+                decode_cbor_data(m.value(), w);   // languages_server_to_client
+                a.close();
+            }
+        }
+        m.close();
+    }
+
     inline void decode_fp(unsigned int fp_type,
                    datum &d,
                    writeable &w) {
@@ -410,6 +527,12 @@ namespace cbor_fingerprint {
         case fingerprint_type_tofsee:
             decode_tofsee_fp(d, w);
             break;
+        case fingerprint_type_stun:
+            decode_stun_fp(d, w);
+            break;
+        case fingerprint_type_ssh:
+            decode_ssh_fp(d, w);
+            break;
         default:
             ;
         }
@@ -433,26 +556,27 @@ namespace cbor_fingerprint {
     // test cbor fingerprint encoding and decoding
     //
     static bool test_fingerprint(const char *fingerprint_string, FILE *f=nullptr) {
-            data_buffer<2048> data_buf;
-            datum fp_data{(uint8_t *)fingerprint_string, (uint8_t *)fingerprint_string + strlen(fingerprint_string)};
-            cbor_fingerprint::encode_cbor_fingerprint(fp_data, data_buf);
-
-            data_buffer<2048> out_buf;
-            datum encoded_data{data_buf.contents()};
-            cbor_fingerprint::decode_cbor_fingerprint(encoded_data, out_buf);
-            if (out_buf.contents().cmp(fp_data) != 0) {
-                if (f) {
-                    fprintf(f, "ERROR: MISMATCH\n");
-                    fprintf(f, "fingerprint:              %s\n", fingerprint_string);
-                    fprintf(f, "CBOR encoded fingerprint: ");
-                    data_buf.contents().fprint_hex(f); fputc('\n', f);
-                    fprintf(f, "decoded fingerprint:      ");
-                    out_buf.contents().fprint(f); fputc('\n', f);
-                    cbor::decode_fprint(data_buf.contents(), f);
-                }
-                return false;
+        data_buffer<2048> data_buf;
+        datum fp_data{(uint8_t *)fingerprint_string, (uint8_t *)fingerprint_string + strlen(fingerprint_string)};
+        cbor_fingerprint::encode_cbor_fingerprint(fp_data, data_buf);
+
+        data_buffer<2048> out_buf;
+        datum encoded_data{data_buf.contents()};
+        cbor_fingerprint::decode_cbor_fingerprint(encoded_data, out_buf);
+
+        if (out_buf.contents().cmp(fp_data) != 0) {
+            if (f) {
+                fprintf(f, "ERROR: MISMATCH\n");
+                fprintf(f, "fingerprint:              %s\n", fingerprint_string);
+                fprintf(f, "CBOR encoded fingerprint: ");
+                data_buf.contents().fprint_hex(f); fputc('\n', f);
+                fprintf(f, "decoded fingerprint:      ");
+                out_buf.contents().fprint(f); fputc('\n', f);
+                cbor::decode_fprint(data_buf.contents(), f);
             }
-            return true;
+            return false;
+        }
+        return true;
     };
 
     // cbor_fingerprint::unit_test() returns `true` if all unit tests
@@ -469,7 +593,10 @@ namespace cbor_fingerprint {
             "quic/(00000001)(0303)(130113021303)[(000a000a00086399001d00170018)(002b0003020304)((0039)[(01)(03)(04)(05)(06)(07)(08)(09)(0f)(1b)(20)(80004752)(80ff73db)])(4469)]",
             "http/randomized",
             "tls/1/randomized",
-            "quic/randomized"
+            "quic/randomized",
+            "stun/1/randomized",
+            "stun/1/(00)(0001)(01)((8022)(0006)(0020)(0008)(8028))",
+            "ssh/(656364682d736861322d6e697374703235362c656364682d736861322d6e697374703338342c656364682d736861322d6e697374703532312c6469666669652d68656c6c6d616e2d67726f757031342d736861312c6469666669652d68656c6c6d616e2d67726f75702d65786368616e67652d7368613235362c6469666669652d68656c6c6d616e2d67726f75702d65786368616e67652d736861312c6469666669652d68656c6c6d616e2d67726f7570312d73686131)(7373682d7273612c7373682d6473732c65636473612d736861322d6e697374703235362c65636473612d736861322d6e697374703338342c65636473612d736861322d6e69737470353231)(6165733132382d6374722c6165733132382d6362632c336465732d6374722c336465732d6362632c626c6f77666973682d6362632c6165733139322d6374722c6165733139322d6362632c6165733235362d6374722c6165733235362d636263)(6165733132382d6374722c6165733132382d6362632c336465732d6374722c336465732d6362632c626c6f77666973682d6362632c6165733139322d6374722c6165733139322d6362632c6165733235362d6374722c6165733235362d636263)(686d61632d6d64352c686d61632d736861312c686d61632d736861322d3235362c686d61632d736861312d39362c686d61632d6d64352d3936)(686d61632d6d64352c686d61632d736861312c686d61632d736861322d3235362c686d61632d736861312d39362c686d61632d6d64352d3936)(6e6f6e65)(6e6f6e65)()()"
         };
         bool all_tests_passed = true;
         for (const auto & fp_str : fps) {
