Merge pull request #504 from network-intelligence/fdc-json-fix

Fixing `get_json_decoded_fdc()` to no longer assume UTF-8 strings in `cbor::text_string` elements

Author: davidmcgrew

doc/CHANGELOG.md

@@ -1,5 +1,6 @@
 # CHANGELOG for Mercury
 
+* Fixed `get_json_decoded_fdc()` to not assume UTF-8 inputs.
 * Added CBOR encoding/decoding for SSH and STUN fingerprints.
 * Fixing compiler warnings related to ABI differences
 * CMake changes required to add xsimd as submodule and fixing

src/cbor.cpp

@@ -87,30 +87,10 @@ int main(int argc, char *argv[]) {
             }
         }
         if (decode_fdc) {
-            static const size_t MAX_FP_STR_LEN     = 8192;
-            char fp_str[MAX_FP_STR_LEN];
-            char dst_ip_str[MAX_ADDR_STR_LEN];
-            char sn_str[MAX_SNI_LEN];
-            char ua_str[MAX_USER_AGENT_LEN];
-            uint16_t dst_port;
-            uint64_t truncation;
-
-            bool ok = fdc::decode(input,
-                                  writeable{(uint8_t*)fp_str, MAX_FP_STR_LEN},
-                                  writeable{(uint8_t*)sn_str, MAX_SNI_LEN},
-                                  writeable{(uint8_t*)dst_ip_str, MAX_ADDR_STR_LEN},
-                                  dst_port,
-                                  writeable{(uint8_t*)ua_str, MAX_USER_AGENT_LEN},
-                                  truncation);
-            if (ok) {
-                fprintf(stdout, "{\"fdc\":{");
-                fprintf(stdout, "\"fingerprint\": \"%s\",", fp_str);
-                fprintf(stdout, "\"sni\": \"%s\",", sn_str);
-                fprintf(stdout, "\"dst_ip_str\": \"%s\",", dst_ip_str);
-                fprintf(stdout, "\"dst_port\": %u,", dst_port);
-                fprintf(stdout, "\"user-agent\": \"%s\",", ua_str);
-                fprintf(stdout, "\"truncation\": %lu", truncation);
-                fprintf(stdout, "}}\n");
+
+            std::string json_string = get_json_decoded_fdc((const char *)input.data, input.length());
+            if (json_string != "") {
+                fprintf(stdout , "%s\n", json_string.c_str());
             } else {
                 fprintf(stderr, "error: could not decode FDC\n");
             }

src/libmerc/CMakeLists.txt

@@ -318,8 +318,8 @@ set(cbor_src
     ../cbor.cpp
 )
 add_executable(cbor ${cbor_src})
-target_compile_options(cbor PUBLIC ${LIBMERC_COMPILE_OPT})
-target_link_libraries(cbor ZLIB::ZLIB)
+target_compile_options(cbor PUBLIC ${LIBMERC_COMPILE_OPT} -UDONT_USE_STDERR)
+target_link_libraries(cbor ZLIB::ZLIB )
 # }
 
 

src/libmerc/fdc.hpp

@@ -739,7 +739,7 @@ class fdc {
         //
         const char *tls_fp = "tls/1/(0301)(c014c00a00390038c00fc0050035c012c00800160013c00dc003000ac013c00900330032c00ec004002fc011c007c00cc002000500040015001200090014001100080006000300ff)[(0000)(000a00340032000100020003000400050006000700080009000a000b000c000d000e000f0010001100120013001400150016001700180019)(000b000403000102)(0023)]";
         const char *http_fp = "http/(434f4e4e454354)(485454502f312e31)((486f7374)(557365722d4167656e74))";
-        static constexpr size_t num_tests = 4;
+        static constexpr size_t num_tests = 5;
         fdc fdc_object[num_tests]{
             {
                 datum{tls_fp},
@@ -772,7 +772,15 @@ class fdc {
                 "72.163.217.105",
                 80,
                 truncation_status::reassembled_truncated
-            }
+            },
+            {
+                datum{http_fp},
+                "user-agent with utf8: stra\u00DFe \r\n\"",
+                "abc.com",
+                "72.163.217.105",
+                80,
+                truncation_status::reassembled_truncated
+            },
         };
         for (size_t i = 0; i < num_tests; i++){
 
@@ -785,16 +793,15 @@ class fdc {
 
             // decode the data in the buffer to decoded_fdc
             //
-            static const size_t MAX_FP_STR_LEN     = 8192;
-            char fp_str[MAX_FP_STR_LEN];
+            char fp_str[fingerprint::MAX_FP_STR_LEN];
             char dst_ip_str[MAX_ADDR_STR_LEN];
             char sn_str[MAX_SNI_LEN];
             char ua_str[MAX_USER_AGENT_LEN];
             uint16_t dst_port;
             uint64_t truncation;
 
             bool decoding_ok = fdc::decode(encoded_fdc,
-                                           writeable{(uint8_t*)fp_str, MAX_FP_STR_LEN},
+                                           writeable{(uint8_t*)fp_str, fingerprint::MAX_FP_STR_LEN},
                                            writeable{(uint8_t*)sn_str, MAX_SNI_LEN},
                                            writeable{(uint8_t*)dst_ip_str, MAX_ADDR_STR_LEN},
                                            dst_port,
@@ -840,42 +847,41 @@ class fdc {
 
 [[maybe_unused]] static std::string get_json_decoded_fdc(const char *fdc_blob, ssize_t blob_len) {
     datum fdc_data = datum{(uint8_t*)fdc_blob,(uint8_t*)(fdc_blob+blob_len)};
-    static const size_t MAX_FP_STR_LEN     = 8192;
-    char fp_str[MAX_FP_STR_LEN];
+    char fp_str[fingerprint::MAX_FP_STR_LEN];
     char dst_ip_str[MAX_ADDR_STR_LEN];
     char sn_str[MAX_SNI_LEN];
     char ua_str[MAX_USER_AGENT_LEN];
     uint16_t dst_port;
     uint64_t truncation;
 
-    char buffer[8192];
+    char buffer[10240];
     struct buffer_stream buf_json(buffer, sizeof(buffer));
     struct json_object record(&buf_json);
 
     bool ok = fdc::decode(fdc_data,
-                          writeable{(uint8_t*)fp_str, MAX_FP_STR_LEN},
+                          writeable{(uint8_t*)fp_str, fingerprint::MAX_FP_STR_LEN},
                           writeable{(uint8_t*)sn_str, MAX_SNI_LEN},
                           writeable{(uint8_t*)dst_ip_str, MAX_ADDR_STR_LEN},
                           dst_port,
                           writeable{(uint8_t*)ua_str, MAX_USER_AGENT_LEN},
                           truncation);
     if (ok) {
         json_object fdc_json(record,"fdc");
-        fdc_json.print_key_string("fingerprint",fp_str);
-        fdc_json.print_key_string("sni",sn_str);
-        fdc_json.print_key_string("dst_ip_str",dst_ip_str);
-        fdc_json.print_key_int("dst_port",dst_port);
-        fdc_json.print_key_string("user_agent",ua_str);
-        fdc_json.print_key_string("truncation",get_truncation_str(((truncation_status)truncation)));
+        fdc_json.print_key_string("fingerprint", fp_str);
+        fdc_json.print_key_json_string("sni", datum{sn_str});
+        fdc_json.print_key_json_string("dst_ip_str", datum{dst_ip_str});
+        fdc_json.print_key_int("dst_port", dst_port);
+        fdc_json.print_key_json_string("user_agent", datum{ua_str});
+        fdc_json.print_key_string("truncation", get_truncation_str(((truncation_status)truncation)));
         fdc_json.close();
         record.close();
         buf_json.write_char('\0');  // null terminate
         return buf_json.get_string();
-        
+
     } else {
         return "";
     }
-   
+
 }
 
 

src/libmerc/fingerprint.h

@@ -11,8 +11,10 @@
 #include "libmerc.h"  // for fingerprint_type
 
 class fingerprint {
-    enum fingerprint_type type;
+public:
     static const size_t MAX_FP_STR_LEN = 8192;
+private:
+    enum fingerprint_type type;
     char fp_str[MAX_FP_STR_LEN];
     struct buffer_stream fp_buf;