Merge pull request #466 from network-intelligence/dev

Merging dev into trunk

Author: davidmcgrew

VERSION

@@ -1 +1 @@
-2.7.0
+2.7.1

doc/CHANGELOG.md

@@ -1,9 +1,13 @@
 # CHANGELOG for Mercury
-* Added a new configuration option, minimize-ram, which reduces
-* the RAM usage of mercury library when enabled
 
 ## VERSION 2.7.1
 * Updated QUIC reassembly logic for reordered QUIC crypto frames
+* Refactored the IP subnet reading code to minimize the amount of
+  temporary RAM needed
+* Added a new configuration option, minimize-ram, which reduces
+  the RAM usage of mercury library when enabled
+* Added changes to allow classifier to use custom weights
+  and introduced new cython interface
 
 ## VERSION 2.7.0
 * Added minimal RDP (Remote Desktop Protocol) support, which

src/cython/_version.py

@@ -1 +1 @@
-__version__ = '2.7.0'
+__version__ = '2.7.1'

src/cython/mercury.pyx

@@ -31,7 +31,7 @@ from cython.operator import dereference
 #   CC=g++ CXX=g++ python setup.py install
 
 # TODO: actually handle version
-__version__ = '2.7.0'
+__version__ = '2.7.1'
 
 # imports from mercury's dns
 cdef extern from "../libmerc/dns.h":
@@ -116,10 +116,10 @@ cdef extern from "../libmerc/analysis.h":
     cdef cppclass classifier:
         analysis_result perform_analysis(const char *fp_str, const char *server_name, const char *dst_ip, uint16_t dst_port, const char *user_agent)
 
-        # analysis_result perform_analysis_with_weights(const char *fp_str, const char *server_name, const char *dst_ip, uint16_t dst_port, const char *user_agent,
-        #                         long double new_as_weight, long double new_domain_weight,
-        #                         long double new_port_weight, long double new_ip_weight,
-        #                         long double new_sni_weight, long double new_ua_weight)
+        analysis_result perform_analysis_with_weights(const char *fp_str, const char *server_name, const char *dst_ip, uint16_t dst_port, const char *user_agent,
+                                 double new_as_weight, double new_domain_weight,
+                                 double new_port_weight, double new_ip_weight,
+                                 double new_sni_weight, double new_ua_weight)
 
 
 cdef extern from "../libmerc/watchlist.hpp":
@@ -435,73 +435,72 @@ cdef class Mercury:
 
         return result
 
-    # cpdef dict perform_analysis_with_weights(self, str fp_str, str server_name, str dst_ip, int dst_port, str user_agent,
-    #                              long double new_as_weight, long double new_domain_weight,
-    #                              long double new_port_weight, long double new_ip_weight,
-    #                              long double new_sni_weight, long double new_ua_weight):
-    #     """
-    #     Directly call into mercury analysis functionality by providing all needed data features. Additionally,
-    #     supply custom weights for each data feature.
-    #
-    #     :param fp_str: mercury-generated network protocol fingerprint
-    #     :type fp_str: str
-    #     :param server_name: The visible, fully qualified domain name, found in the server_name extension or the HTTP Host field
-    #     :type server_name: str
-    #     :param dst_ip: The destination IP address associated with the packet of interest
-    #     :type dst_ip: str
-    #     :param dst_port: The destination port associated with the packet of interest
-    #     :type dst_port: int
-    #     :param user_agent: If analyzing an HTTP packet, provide the contents of the HTTP User-Agent field
-    #     :type user_agent: str
-    #     :param new_as_weight: Updated weight for the Autonomous System data feature
-    #     :type new_as_weight: long double
-    #     :param new_domain_weight: Updated weight for the domain name data feature
-    #     :type new_domain_weight: long double
-    #     :param new_port_weight: Updated weight for the destination port data feature
-    #     :type new_port_weight: long double
-    #     :param new_ip_weight: Updated weight for the destination IP address data feature
-    #     :type new_ip_weight: long double
-    #     :param new_sni_weight: Updated weight for the server_name data feature
-    #     :type new_sni_weight: long double
-    #     :param new_ua_weight: Updated weight for the User-Agent data feature
-    #     :type new_ua_weight: long double
-    #     :return: JSON-encoded analysis output
-    #     :rtype: dict
-    #     """
-    #     if not self.do_analysis:
-    #         print(f'error: classifier not loaded (is do_analysis set to True?)')
-    #         return None
-    #
-    #     cdef bytes fp_str_b = fp_str.encode()
-    #     cdef char* fp_str_c = fp_str_b
-    #     cdef bytes server_name_b = server_name.encode()
-    #     cdef char* server_name_c = server_name_b
-    #     cdef bytes dst_ip_b = dst_ip.encode()
-    #     cdef char* dst_ip_c = dst_ip_b
-    #     if user_agent == None:
-    #         user_agent = 'None'
-    #     cdef bytes user_agent_b = user_agent.encode()
-    #     cdef char* user_agent_c = user_agent_b
-    #     if user_agent == 'None':
-    #         user_agent_c = NULL
-    #
-    #     cdef analysis_result ar = self.clf.perform_analysis_with_weights(fp_str_c, server_name_c, dst_ip_c, dst_port, user_agent_c,
-    #                                                 new_as_weight, new_domain_weight, new_port_weight,
-    #                                                 new_ip_weight, new_sni_weight, new_ua_weight)
-    #
-    #     cdef fingerprint_status fp_status_enum = ar.status
-    #     fp_status = fp_status_dict[fp_status_enum]
-    #
-    #     cdef dict result = {}
-    #     result['fingerprint_info'] = {}
-    #     result['fingerprint_info']['status'] = fp_status
-    #     result['analysis'] = {}
-    #     result['analysis']['process']   = ar.max_proc.decode('UTF-8')
-    #     result['analysis']['score']     = ar.max_score
-    #     result['analysis']['malware']   = ar.max_mal
-    #     result['analysis']['p_malware'] = ar.malware_prob
-    #
-    #     return result
+    cpdef dict perform_analysis_with_weights(self, str fp_str, str server_name, str dst_ip, int dst_port, str user_agent,
+                                             double new_as_weight, double new_domain_weight,
+                                             double new_port_weight, double new_ip_weight,
+                                             double new_sni_weight, double new_ua_weight):
+         """
+         Directly call into mercury analysis functionality by providing all needed data features. Additionally,
+         supply custom weights for each data feature.
+ 
+         :param fp_str: mercury-generated network protocol fingerprint
+         :type fp_str: str
+         :param server_name: The visible, fully qualified domain name, found in the server_name extension or the HTTP Host field
+         :type server_name: str
+         :param dst_ip: The destination IP address associated with the packet of interest
+         :type dst_ip: str
+         :param dst_port: The destination port associated with the packet of interest
+         :type dst_port: int
+         :param user_agent: If analyzing an HTTP packet, provide the contents of the HTTP User-Agent field
+         :type user_agent: str
+         :param new_as_weight: Updated weight for the Autonomous System data feature
+         :type new_as_weight: long double
+         :param new_domain_weight: Updated weight for the domain name data feature
+         :type new_domain_weight: long double
+         :param new_port_weight: Updated weight for the destination port data feature
+         :type new_port_weight: long double
+         :param new_ip_weight: Updated weight for the destination IP address data feature
+         :type new_ip_weight: long double
+         :param new_sni_weight: Updated weight for the server_name data feature
+         :type new_sni_weight: long double
+         :param new_ua_weight: Updated weight for the User-Agent data feature
+         :type new_ua_weight: long double
+         :return: JSON-encoded analysis output
+         :rtype: dict
+         """
+         if not self.do_analysis:
+             print(f'error: classifier not loaded (is do_analysis set to True?)')
+             return None
+ 
+         cdef bytes fp_str_b = fp_str.encode()
+         cdef char* fp_str_c = fp_str_b
+         cdef bytes server_name_b = server_name.encode()
+         cdef char* server_name_c = server_name_b
+         cdef bytes dst_ip_b = dst_ip.encode()
+         cdef char* dst_ip_c = dst_ip_b
+         if user_agent == None:
+             user_agent = 'None'
+         cdef bytes user_agent_b = user_agent.encode()
+         cdef char* user_agent_c = user_agent_b
+
+         cdef analysis_result ar = self.clf.perform_analysis_with_weights(
+                                                     fp_str_c, server_name_c, dst_ip_c, dst_port, user_agent_c,
+                                                     new_as_weight, new_domain_weight, new_port_weight,
+                                                     new_ip_weight, new_sni_weight, new_ua_weight)
+
+         cdef fingerprint_status fp_status_enum = ar.status
+         fp_status = fp_status_dict[fp_status_enum]
+
+         cdef dict result = {}
+         result['fingerprint_info'] = {}
+         result['fingerprint_info']['status'] = fp_status
+         result['analysis'] = {}
+         result['analysis']['process']   = ar.max_proc.decode('UTF-8')
+         result['analysis']['score']     = ar.max_score
+         result['analysis']['malware']   = ar.max_mal
+         result['analysis']['p_malware'] = ar.malware_prob
+
+         return result
 
     cdef list extract_attributes(self, analysis_result ar):
         cdef char tags_buf[8192]
@@ -551,8 +550,6 @@ cdef class Mercury:
             user_agent = 'None'
         cdef bytes user_agent_b = user_agent.encode()
         cdef char* user_agent_c = user_agent_b
-        if user_agent == 'None':
-            user_agent_c = NULL
 
         cdef analysis_result ar = self.clf.perform_analysis(fp_str_c, server_name_c, dst_ip_c, dst_port, user_agent_c)
 

src/libmerc/addr.cc

@@ -93,28 +93,46 @@ uint32_t subnet_data::get_asn_info(const char* dst_ip) const {
     return 0;
 }
 
-int subnet_data::process_line(std::string &line_str) {
+int subnet_data::process_asn_subnets(const std::vector<std::string> &subnets) {
 
-    // set the prefix[num] to the subnet and ASN found in line
-    if (lct_subnet_set_from_string(&prefix[num], line_str.c_str()) != 0) {
-        printf_err(log_err, "could not parse subnet string '%s'\n", line_str.c_str());
-        return -1;  // failure
+    prefix = (lct_subnet_t *)calloc(sizeof(lct_subnet_t), subnets.size());
+    if (prefix == nullptr) {
+        throw std::runtime_error("error: could not initialize subnet_data");
+    }
+
+    // Add Special and Private subnets to ASN subnets
+    //
+    // parse the subnets and ASN strings    // start with the RFC 1918 and 3927 private and link local
+    // subnets as a basis for any table set
+    //
+    // num += init_private_subnets(&prefix[num], BGP_MAX_ENTRIES);
+    //
+    // fill up the rest of the array with reserved IP subnets
+    //
+    // num += init_special_subnets(&prefix[num], BGP_MAX_ENTRIES);
+
+    for (const std::string &line_str : subnets) {
+        // set the prefix[num] to the subnet and ASN found in line
+        if (lct_subnet_set_from_string(&prefix[num], line_str.c_str()) != 0) {
+            printf_err(log_err, "could not parse subnet string '%s'\n", line_str.c_str());
+            return -1;  // failure
+        }
+        num++;
     }
-    num++;
     return 0;       // success
 }
 
 int subnet_data::lct_add_domain_mapping(uint32_t &addr, uint8_t &mask_length, std::string &domain_name, std::unordered_map<uint32_t, ssize_t> &subnet_map) {
     uint32_t domain_idx;
-    if (domains_watchlist.find(domain_name) == domains_watchlist.end()) {    // new domain; assing a domain id and save in the domain watchlist
+    if (domains_watchlist.find(domain_name) == domains_watchlist.end()) {    // new domain; assign a domain id and save in the domain watchlist
         domain_idx = domains_watchlist.size();
         domains_watchlist[domain_name] = domain_idx;
     } else {
-        domain_idx = domains_watchlist[domain_name];    // domain already see; retrieve domain id
+        domain_idx = domains_watchlist[domain_name];    // domain already seen; retrieve domain id
     }
 
     lct_subnet<uint32_t> *subnet_itr;
-    if (subnet_map.find(addr) != subnet_map.end()) {    // subnet presetn in map, domain_idx needs to be appended
+    if (subnet_map.find(addr) != subnet_map.end()) {    // subnet present in map, domain_idx needs to be appended
         subnet_itr = &domains_prefix[subnet_map[addr]];
         if (subnet_itr->info.type == IP_SUBNET_DOMAIN && subnet_itr->addr == addr && subnet_itr->len == mask_length) {
             uint8_t *old_arr = subnet_itr->info.domain.domain_idx_arr;
@@ -150,58 +168,67 @@ int subnet_data::lct_add_domain_mapping(uint32_t &addr, uint8_t &mask_length, st
     return 0;       // success
 }
 
-int subnet_data::process_domain_mappings_line(std::string &line_str, std::unordered_map<uint32_t, ssize_t> &subnet_map) {
-    rapidjson::Document domain_obj;
-    domain_obj.Parse(line_str.c_str());
-    if(!domain_obj.IsObject()) {
-        printf_err(log_warning, "invalid JSON line in resource file\n");
-        return -1;
+int subnet_data::process_domain_mapping_subnets(const std::vector<std::string> &subnets) {
+
+    std::unordered_map<uint32_t, ssize_t> subnet_map;
+    domains_prefix = (lct_subnet_t *)calloc(sizeof(lct_subnet_t), subnets.size());
+    if (domains_prefix == nullptr) {
+        throw std::runtime_error("error: could not initialize domains_prefix");
     }
 
-    std::string subnet_type;
-    std::string subnet_str;
-    std::string subnet_tag;
-    
-    uint32_t addr;
-    unsigned char *dq = (unsigned char *)&addr;
-    uint8_t mask_length;
-    constexpr unsigned int bits_in_T = sizeof(uint32_t) * 8;
+    for (const std::string &line_str : subnets) { 
+        rapidjson::Document domain_obj;
+        domain_obj.Parse(line_str.c_str());
+        if(!domain_obj.IsObject()) {
+            printf_err(log_warning, "invalid JSON line in resource file\n");
+            return -1;
+        }
 
-    if (domain_obj.HasMember("subnet") && domain_obj["subnet"].IsString()) {
-        subnet_str = domain_obj["subnet"].GetString();
-    }
-    else {
-        return -1;
-    }
-    if (domain_obj.HasMember("type") && domain_obj["type"].IsString()) {
-        subnet_type = domain_obj["type"].GetString();
-    }
-    else {
-        return -1;
-    }
-    if (domain_obj.HasMember("tag") && domain_obj["tag"].IsString()) {
-        subnet_tag = domain_obj["tag"].GetString();
-    }
-    else {
-        return -1;
-    }
+        std::string subnet_type;
+        std::string subnet_str;
+        std::string subnet_tag;
+        
+        uint32_t addr;
+        unsigned char *dq = (unsigned char *)&addr;
+        uint8_t mask_length;
+        constexpr unsigned int bits_in_T = sizeof(uint32_t) * 8;
 
-    if (subnet_type == "domain_mapping") {
-        int num_items_parsed = sscanf(subnet_str.c_str(),"%hhu.%hhu.%hhu.%hhu/%hhu",
-            dq + 3, dq + 2, dq + 1, dq, &mask_length);
-        if (num_items_parsed == 5) {    // invalid IP or IPv6
-            if ((mask_length == 0) || (mask_length > bits_in_T)) {
-                fprintf(stderr, "ERROR: %u is not a valid prefix length\n", mask_length);
-                return -1;      // failure
-            }
+        if (domain_obj.HasMember("subnet") && domain_obj["subnet"].IsString()) {
+            subnet_str = domain_obj["subnet"].GetString();
+        }
+        else {
+            return -1;
+        }
+        if (domain_obj.HasMember("type") && domain_obj["type"].IsString()) {
+            subnet_type = domain_obj["type"].GetString();
+        }
+        else {
+            return -1;
+        }
+        if (domain_obj.HasMember("tag") && domain_obj["tag"].IsString()) {
+            subnet_tag = domain_obj["tag"].GetString();
+        }
+        else {
+            return -1;
+        }
 
-            if (lct_add_domain_mapping(addr, mask_length, subnet_tag, subnet_map) != 0) {
-                return -1;      // failure
+        if (subnet_type == "domain_mapping") {
+            int num_items_parsed = sscanf(subnet_str.c_str(),"%hhu.%hhu.%hhu.%hhu/%hhu",
+                dq + 3, dq + 2, dq + 1, dq, &mask_length);
+            if (num_items_parsed == 5) {    // invalid IP or IPv6
+                if ((mask_length == 0) || (mask_length > bits_in_T)) {
+                    fprintf(stderr, "ERROR: %u is not a valid prefix length\n", mask_length);
+                    return -1;      // failure
+                }
+
+                if (lct_add_domain_mapping(addr, mask_length, subnet_tag, subnet_map) != 0) {
+                    return -1;      // failure
+                }
             }
         }
-    }
-    else if (subnet_type == "proxy" || subnet_type == "sinkhole") {
-        domain_faking_exceptions.insert(addr);
+        else if (subnet_type == "proxy" || subnet_type == "sinkhole") {
+            domain_faking_exceptions.insert(addr);
+        }
     }
 
     return 0;