Merge branch 'dev' into new-cbor-fps

davidmcgrew · GitHub Enterprise · commit df34f7c2aa29 · 2025-08-08T06:48:48.000-04:00
diff --git a/Makefile_helper.mk.in b/Makefile_helper.mk.in
@@ -58,6 +58,7 @@ CFLAGS += -Wno-deprecated-declarations
 CFLAGS += -Wno-long-long
 CFLAGS += -Wmissing-noreturn
 CFLAGS += -Wunreachable-code
+CFLAGS += -Wno-psabi
 CFLAGS += -fvisibility=hidden
 CFLAGS += -DNDEBUG
 # CFLAGS += -g
diff --git a/doc/CHANGELOG.md b/doc/CHANGELOG.md
@@ -1,6 +1,7 @@
 # CHANGELOG for Mercury
 
 * Added CBOR encoding/decoding for SSH and STUN fingerprints.
+* Fixing compiler warnings related to ABI differences
 * CMake changes required to add xsimd as submodule and fixing
   windows compilation issues
 * Removed duplicate UTF-8 and IP address output code used in
diff --git a/src/libmerc/flow_key.h b/src/libmerc/flow_key.h
@@ -140,15 +140,12 @@ struct key {
         return addr.ipv6.dst.is_global();
     }
 
-    // write out the (optionally normalized) destination address
+    // write out the destination address
     //
-    void sprintf_dst_addr(char *dst_addr_str, bool norm=true) const {
+    void sprintf_dst_addr(char *dst_addr_str) const {
 
         if (ip_vers == 4) {
             ipv4_address tmp_addr{addr.ipv4.dst};
-            if (norm) {
-                normalize(tmp_addr);
-            }
             uint8_t *d = (uint8_t *)&tmp_addr;
             snprintf(dst_addr_str,
                      MAX_ADDR_STR_LEN,
@@ -157,9 +154,6 @@ struct key {
 
         } else if (ip_vers == 6) {
             ipv6_address tmp_addr{addr.ipv6.dst};
-            if (norm) {
-                normalize(tmp_addr);
-            }
             uint8_t *d = (uint8_t *)&tmp_addr;
             sprintf_ipv6_addr(dst_addr_str, d);
         } else {
diff --git a/src/libmerc/ip_address.hpp b/src/libmerc/ip_address.hpp
@@ -205,6 +205,9 @@ bool ipv4_address::unit_test(FILE *output) {  // output=nullptr by default
     return all_passed;
 }
 
+
+using ipv6_array_t = std::array<uint8_t, 16>;
+
 /// an IP version six address in network byte order.  This class
 /// represents a raw (binary) address; to parse a textual
 /// representation of an IPv6 address, use \ref ipv6_address_string.
@@ -307,11 +310,7 @@ struct ipv6_address {
         multicast
     };
 
-    // bool is_global_unicast() const {
-    //     return (a & 0xe0000000) == 0x20000000;
-    // }
     bool is_global_unicast() const {
-        // fprintf(stderr, "check: %08x\t%08x==%08x\n", a, (a & hton<uint32_t>(0xe0000000)),  hton<uint32_t>(0x20000000));
         return (a[0] & hton<uint32_t>(0xe0000000)) == hton<uint32_t>(0x20000000);
     }
     bool is_unique_local_unicast() const {
@@ -327,7 +326,7 @@ struct ipv6_address {
         return (a[0] & hton<uint32_t>(0xff000000)) == hton<uint32_t>(0xff000000);
     }
     bool is_global() const {
-        return is_global_unicast();   // TODO: consider global multicast
+        return is_global_unicast() || is_ipv4_mapped();
     }
     bool is_ipv4_mapped() const {
         return (a[0] == 0 && a[1] == 0 && a[2] == hton<uint32_t>(0x0000ffff));
@@ -337,6 +336,20 @@ struct ipv6_address {
 
 };
 
+// hasher for ipv6_address
+//
+namespace std {
+    template <>
+    struct hash<ipv6_address> {
+        std::size_t operator()(const ipv6_address& addr) const {
+            return std::hash<uint32_t>{}(addr.a[0])
+                  ^ std::hash<uint32_t>{}(addr.a[1])
+                  ^ std::hash<uint32_t>{}(addr.a[2])
+                  ^ std::hash<uint32_t>{}(addr.a[3]);
+        }
+    };
+}
+
 inline bool ipv6_address::unit_test() {
 
     // ipv6_address addr;
@@ -384,16 +397,18 @@ namespace normalized {
     static const ipv6_address ipv6_unique_local{0x000000fd, 0x00000000, 0x00000000, 0x01000000 };
 };
 
-inline void normalize(ipv4_address &a) {
+inline ipv4_address normalize(const ipv4_address &a) {
     if (!a.is_global()) {
-        a = normalized::ipv4_private_use;
+        return normalized::ipv4_private_use;
     }
+    return a;
 }
 
-inline void normalize(ipv6_address &a) {
+inline ipv6_address normalize(const ipv6_address &a) {
     if (!a.is_global()) {
-        a = normalized::ipv6_unique_local;
+        return normalized::ipv6_unique_local;
     }
+    return a;
 }
 
 struct ip_address {
@@ -543,8 +558,6 @@ T hex_str_to_uint(const hex_digits &d) {
 }
 
 
-using ipv6_array_t = std::array<uint8_t, 16>;
-
 static inline void ipv6_array_print(FILE *f, ipv6_array_t ipv6) {
     fprintf(f,
             "%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x",
@@ -865,6 +878,11 @@ class ipv6_address_string {
         return x;
     }
 
+    ipv6_address get_address() const {
+        ipv6_array_t arry = get_value_array();
+        return get_ipv6_address(arry);
+    }
+
     // unit_test() is a static function that performs a unit test of
     // this class, using the example addresses from RFC 4291.  It
     // returns true if all tests pass, and false otherwise.
diff --git a/src/libmerc/naive_bayes.hpp b/src/libmerc/naive_bayes.hpp
@@ -227,7 +227,7 @@ inline auto feature<uint64_t>::convert(const char *s) -> uint64_t {
 ///
 class ip_addr_feature {
     std::unordered_map<uint32_t, std::vector<class update>> ipv4_updates;
-    std::unordered_map<ipv6_array_t, std::vector<class update>> ipv6_updates;
+    std::unordered_map<ipv6_address, std::vector<class update>> ipv6_updates;
 
 public:
     std::string json_name;
@@ -278,15 +278,15 @@ class ip_addr_feature {
         class update u{ (unsigned int)process_index, (log((floating_point_type)count / total_count) - base_prior) * weight };
 
         if (lookahead<ipv4_address_string> ipv4{datum{feature_value}}) {
-            uint32_t addr = ipv4.value.get_value();
-            auto update = ipv4_updates.find(addr);
+            ipv4_address addr = normalize(ipv4.value.get_value());
+            auto update = ipv4_updates.find(addr.get_value());
             if (update != ipv4_updates.end()) {
                 update->second.push_back(u);
             } else {
-                ipv4_updates[addr] = { u };
+                ipv4_updates[addr.get_value()] = { u };
             }
         } else if (lookahead<ipv6_address_string> ipv6{datum{feature_value}}) {
-            ipv6_array_t addr = ipv6.value.get_value_array();
+            ipv6_address addr = normalize(ipv6.value.get_address());
             auto update = ipv6_updates.find(addr);
             if (update != ipv6_updates.end()) {
                 update->second.push_back(u);
@@ -300,14 +300,15 @@ class ip_addr_feature {
     //
     void update(std::vector<floating_point_type> &prob_vector, const std::string &dst_ip_str) const {
         if (lookahead<ipv4_address_string> ipv4{datum{dst_ip_str}}) {
-            auto ip_ip_update = ipv4_updates.find(ipv4.value.get_value());
+            ipv4_address addr = normalize(ipv4.value.get_value());
+            auto ip_ip_update = ipv4_updates.find(addr.get_value());
             if (ip_ip_update != ipv4_updates.end()) {
                 for (const auto &x : ip_ip_update->second) {
                     prob_vector[x.index] += x.value;
                 }
             }
         } else if (lookahead<ipv6_address_string> ipv6{datum{dst_ip_str}}) {
-            auto ip_ip_update = ipv6_updates.find(ipv6.value.get_value_array());
+            auto ip_ip_update = ipv6_updates.find(normalize(ipv6.value.get_address()));
             if (ip_ip_update != ipv6_updates.end()) {
                 for (const auto &x : ip_ip_update->second) {
                     prob_vector[x.index] += x.value;
@@ -320,14 +321,15 @@ class ip_addr_feature {
 
     void update(std::vector<floating_point_type> &prob_vector, const std::string &dst_ip_str, floating_point_type w) const {
         if (lookahead<ipv4_address_string> ipv4{datum{dst_ip_str}}) {
-            auto ip_ip_update = ipv4_updates.find(ipv4.value.get_value());
+            ipv4_address addr = normalize(ipv4.value.get_value());
+            auto ip_ip_update = ipv4_updates.find(addr.get_value());
             if (ip_ip_update != ipv4_updates.end()) {
                 for (const auto &x : ip_ip_update->second) {
                     prob_vector[x.index] += x.value * (w / weight);
                 }
             }
         } else if (lookahead<ipv6_address_string> ipv6{datum{dst_ip_str}}) {
-            auto ip_ip_update = ipv6_updates.find(ipv6.value.get_value_array());
+            auto ip_ip_update = ipv6_updates.find(normalize(ipv6.value.get_address()));
             if (ip_ip_update != ipv6_updates.end()) {
                 for (const auto &x : ip_ip_update->second) {
                     prob_vector[x.index] += x.value * (w / weight);
diff --git a/src/libmerc/pkt_proc.h b/src/libmerc/pkt_proc.h
@@ -106,7 +106,7 @@ struct stateful_pkt_proc {
     mercury_context m;
     classifier *c;        // TODO: change to reference
     data_aggregator *ag;
-    global_config global_vars;
+    const global_config &global_vars;
     class traffic_selector &selector;
     quic_crypto_engine quic_crypto;
     struct tcp_reassembler *reassembler_ptr = nullptr;
@@ -119,7 +119,7 @@ struct stateful_pkt_proc {
         analysis{},
         mq{nullptr},
         m{mc},
-        c{nullptr},
+        c{mc->c},
         ag{nullptr},
         global_vars{mc->global_vars},
         selector{mc->selector},
@@ -137,11 +137,9 @@ struct stateful_pkt_proc {
 
         // set config and classifier to (refer to) context m
         // analysis requires `do_analysis` & `resources` to be set
-        if (m->c == nullptr && m->global_vars.do_analysis && m->global_vars.resources != nullptr) {
+        if (c == nullptr && global_vars.do_analysis && global_vars.resources != nullptr) {
             throw std::runtime_error("error: classifier pointer is null");
         }
-        this->c = m->c;
-        this->global_vars = m->global_vars;
 
         // setting protocol based configuration option to output the raw features
         set_raw_features(global_vars.raw_features);
@@ -289,26 +287,26 @@ struct stateful_pkt_proc {
 
     bool dump_pkt ();
 
-    void set_raw_features(std::unordered_map<std::string, bool> &raw_features) {
-        if (raw_features["all"] or raw_features["tls"]) {
+    void set_raw_features(const std::unordered_map<std::string, bool> &raw_features) {
+        if (raw_features.at("all") or raw_features.at("tls")) {
             tls_client_hello::set_raw_features(true);
         }
-        
-        if (raw_features["all"] or raw_features["stun"]) {
+
+        if (raw_features.at("all") or raw_features.at("stun")) {
             stun::message::set_raw_features(true);
         }
-        
-        if (raw_features["all"] or raw_features["bittorrent"]) {
+
+        if (raw_features.at("all") or raw_features.at("bittorrent")) {
             bittorrent_dht::set_raw_features(true);
             bittorrent_lsd::set_raw_features(true);
             bittorrent_handshake::set_raw_features(true);
         }
-        
-        if (raw_features["all"] or raw_features["smb"]) {
+
+        if (raw_features.at("all") or raw_features.at("smb")) {
             smb2_packet::set_raw_features(true);
         }
-        
-        if (raw_features["all"] or raw_features["ssdp"]) {
+
+        if (raw_features.at("all") or raw_features.at("ssdp")) {
             ssdp::set_raw_features(true);
         }
     }
diff --git a/src/libmerc/watchlist.hpp b/src/libmerc/watchlist.hpp
@@ -333,7 +333,7 @@ class server_identifier {
                     a += '.';
                 }
                 ipv4_address addr = std::get<uint32_t>(host_id);
-                normalize(addr);
+                addr = normalize(addr);
                 a += addr.get_dns_label();
             }
             a += "address.alt";
@@ -349,8 +349,7 @@ class server_identifier {
                     a += '.';
                 }
                 ipv6_array_t addr = std::get<ipv6_array_t>(host_id);
-                ipv6_address tmp = get_ipv6_address(addr);
-                // normalize(tmp);
+                ipv6_address tmp = normalize(get_ipv6_address(addr));
                 a += tmp.get_dns_label();
             }
             a += "address.alt";
@@ -495,7 +494,7 @@ class server_identifier {
             { "[::ffff:91.222.113.90]:5000", "_5000.--ffff-5bde-715a.address.alt", 5000 },           // IPv6 addr with embedded IPv4 addr, square braces, and port number
             { "2001:db8::2:1", "2001-db8--2-1.address.alt", {} },                                                        // IPv6 addr with zero compression
             { "240d:c000:2010:1a58:0:95fe:d8b7:5a8f", "240d-c000-2010-1a58-0-95fe-d8b7-5a8f.address.alt", {} },          // IPv6 addr without zero compression
-            { "abcd:888::2:1", "abcd-888--2-1.address.alt", {} },                                                        // IPv6 addr that could be confused for server:port
+            { "abcd:888::2:1", "fd00--1.address.alt", {} },                                                              // Non-global IPv6 addr that could be confused for server:port
             { "cisco.com:443", "_443.cisco.com", 443 },                                  // FQDN with port number
             { ":8080", "_8080.missing.alt", 8080 },                                      // missing FQDN with port number
             { "cisco.com.:443", "_443.cisco.com", 443 },                                 // trailing dot with port number
@@ -524,7 +523,7 @@ inline std::string normalize_ip_address(const std::string &s) {
             return "";  // error: trailing data after address
         }
         ipv4_address addr = addr_str.value.get_value();
-        normalize(addr);
+        addr = normalize(addr);
         return addr.get_string();
     }
     if (lookahead<ipv6_address_string> addr_str{d}) {
@@ -533,7 +532,7 @@ inline std::string normalize_ip_address(const std::string &s) {
             return "";  // error: trailing data after address
         }
         ipv6_address addr = get_ipv6_address(addr_str.value.get_value_array());
-        normalize(addr);
+        addr = normalize(addr);
         return addr.get_string();
     }
     return "";  // error: s is neither an ipv4 nor an ipv6 address
diff --git a/test/compare-stats.py b/test/compare-stats.py
@@ -63,7 +63,7 @@ def read_merc_data(in_file):
             continue
 
         src_ip      = r['src_ip']
-        dst_ip      = normalize_address(r['dst_ip'])
+        dst_ip      = r['dst_ip']
         dst_port    = r['dst_port']
         user_agent = ''
         server_name = ''
diff --git a/unit_tests/libmerc_driver_fdc.cc b/unit_tests/libmerc_driver_fdc.cc
@@ -614,7 +614,7 @@ SCENARIO("test mercury_packet_processor_get_analysis_context_fdc for http reques
 
             THEN("FDC should be written to output buffer") {
                 REQUIRE(bytes_written != fdc_return::FDC_WRITE_INSUFFICIENT_SPACE);
-                REQUIRE(bytes_written == 122);
+                REQUIRE(bytes_written == 136);
                 REQUIRE(fdc_buffer_len == max_buffer_allocation);
             }  
             mercury_packet_processor_destruct(mpp);

Original file line number	Diff line number	Diff line change
`@@ -333,7 +333,7 @@ class server_identifier {`
`333`	`333`	`a += '.';`
`334`	`334`	`}`
`335`	`335`	`ipv4_address addr = std::get<uint32_t>(host_id);`
`336`		`- normalize(addr);`
	`336`	`+ addr = normalize(addr);`
`337`	`337`	`a += addr.get_dns_label();`
`338`	`338`	`}`
`339`	`339`	`a += "address.alt";`
`@@ -349,8 +349,7 @@ class server_identifier {`
`349`	`349`	`a += '.';`
`350`	`350`	`}`
`351`	`351`	`ipv6_array_t addr = std::get<ipv6_array_t>(host_id);`
`352`		`- ipv6_address tmp = get_ipv6_address(addr);`
`353`		`- // normalize(tmp);`
	`352`	`+ ipv6_address tmp = normalize(get_ipv6_address(addr));`
`354`	`353`	`a += tmp.get_dns_label();`
`355`	`354`	`}`
`356`	`355`	`a += "address.alt";`
`@@ -495,7 +494,7 @@ class server_identifier {`
`495`	`494`	`{ "[::ffff:91.222.113.90]:5000", "_5000.--ffff-5bde-715a.address.alt", 5000 }, // IPv6 addr with embedded IPv4 addr, square braces, and port number`
`496`	`495`	`{ "2001:db8::2:1", "2001-db8--2-1.address.alt", {} }, // IPv6 addr with zero compression`
`497`	`496`	`{ "240d:c000:2010:1a58:0:95fe:d8b7:5a8f", "240d-c000-2010-1a58-0-95fe-d8b7-5a8f.address.alt", {} }, // IPv6 addr without zero compression`
`498`		`- { "abcd:888::2:1", "abcd-888--2-1.address.alt", {} }, // IPv6 addr that could be confused for server:port`
	`497`	`+ { "abcd:888::2:1", "fd00--1.address.alt", {} }, // Non-global IPv6 addr that could be confused for server:port`
`499`	`498`	`{ "cisco.com:443", "_443.cisco.com", 443 }, // FQDN with port number`
`500`	`499`	`{ ":8080", "_8080.missing.alt", 8080 }, // missing FQDN with port number`
`501`	`500`	`{ "cisco.com.:443", "_443.cisco.com", 443 }, // trailing dot with port number`
`@@ -524,7 +523,7 @@ inline std::string normalize_ip_address(const std::string &s) {`
`524`	`523`	`return ""; // error: trailing data after address`
`525`	`524`	`}`
`526`	`525`	`ipv4_address addr = addr_str.value.get_value();`
`527`		`- normalize(addr);`
	`526`	`+ addr = normalize(addr);`
`528`	`527`	`return addr.get_string();`
`529`	`528`	`}`
`530`	`529`	`if (lookahead<ipv6_address_string> addr_str{d}) {`
`@@ -533,7 +532,7 @@ inline std::string normalize_ip_address(const std::string &s) {`
`533`	`532`	`return ""; // error: trailing data after address`
`534`	`533`	`}`
`535`	`534`	`ipv6_address addr = get_ipv6_address(addr_str.value.get_value_array());`
`536`		`- normalize(addr);`
	`535`	`+ addr = normalize(addr);`
`537`	`536`	`return addr.get_string();`
`538`	`537`	`}`
`539`	`538`	`return ""; // error: s is neither an ipv4 nor an ipv6 address`
Original file line number	Diff line number	Diff line change
`@@ -614,7 +614,7 @@ SCENARIO("test mercury_packet_processor_get_analysis_context_fdc for http reques`
`614`	`614`
`615`	`615`	`THEN("FDC should be written to output buffer") {`
`616`	`616`	`REQUIRE(bytes_written != fdc_return::FDC_WRITE_INSUFFICIENT_SPACE);`
`617`		`- REQUIRE(bytes_written == 122);`
	`617`	`+ REQUIRE(bytes_written == 136);`
`618`	`618`	`REQUIRE(fdc_buffer_len == max_buffer_allocation);`
`619`	`619`	`}`
`620`	`620`	`mercury_packet_processor_destruct(mpp);`