Merge pull request #456 from network-intelligence/dev

Merging dev into trunk

Author: davidmcgrew

README.md

@@ -118,6 +118,7 @@ GENERAL OPTIONS
    --certs-json                          # output certs as JSON, not base64
    --metadata                            # output more protocol metadata in JSON
    --raw-features                        # select protocols to write out raw features string(see --help)
+   --minimize-ram                        # minimize the ram usage of mercury library
    [-v or --verbose]                     # additional information sent to stderr
    --license                             # write license information to stdout
    --version                             # write version information to stdout
@@ -190,6 +191,8 @@ DETAILS
       tls.certificates  TLS serverCertificates
       tofsee            Tofsee malware communication
       wireguard         WG handshake initiation message
+      geneve            Geneve encapsulation
+      vxlan             VXLAN encapsualtion
       all               all of the above
       <no option>       all of the above
       none              none of the above
@@ -258,6 +261,9 @@ DETAILS
        none            None of the above
       <no option>     None of the above
 
+   --minimize-ram minimizes the ram usage of mercury library by reducing classifer
+   features and minimizing the maximum reassembly segments."
+
    [-v or --verbose] writes additional information to the standard error,
    including the packet count, byte count, elapsed time and processing rate, as
    well as information about threads and files.

VERSION

@@ -1 +1 @@
-2.6.5
+2.7.0

doc/CHANGELOG.md

@@ -1,4 +1,24 @@
 # CHANGELOG for Mercury
+* Added a new configuration option, minimize-ram, which reduces
+* the RAM usage of mercury library when enabled
+
+## VERSION 2.7.1
+* Updated QUIC reassembly logic for reordered QUIC crypto frames
+
+## VERSION 2.7.0
+* Added minimal RDP (Remote Desktop Protocol) support, which
+  reports information about handshakes, security negotiation, and
+  cookies.
+* Added minimal VNC/RFB (Virtual Network Computing / Remote Frame
+  Buffer) support, which reports handshakes and versions.
+* Added MySQL Login support, to report on exposed credentials.
+* Added TACACS+ support, which reports on both `encrypted` and
+  `unencrypted` messages.  Details of unencrypted authenticationd
+  messages are reported in JSON.
+* Added minimal TFTP support, which reports file names and modes.
+* Extended FTP command channel to multi-line responses.
+* Support for reporting outer tunnel parameters and also includes
+  support for PPoE, VXLAN encapsulation and IP encapsulations.
 
 ## VERSION 2.6.5
 * Added support for mutli-line FTP responses.

doc/schema.md

@@ -206,6 +206,10 @@ static enum status mercury_config_parse_line(struct mercury_config *cfg,
         additional_args = str_append(additional_args, ";");
         return status_ok;
 
+    } else if ((arg = command_get_argument("minimize-ram", line)) != NULL) {
+        additional_args = str_append(additional_args, "minimize-ram;");
+        return status_ok;
+
     } else {
         if (line[0] == '#') { /* comment line */
             return status_ok;

src/config.c

@@ -1 +1 @@
-__version__ = '2.6.4'
+__version__ = '2.7.0'

src/cython/_version.py

@@ -31,7 +31,7 @@ from cython.operator import dereference
 #   CC=g++ CXX=g++ python setup.py install
 
 # TODO: actually handle version
-__version__ = '2.6.4'
+__version__ = '2.7.0'
 
 # imports from mercury's dns
 cdef extern from "../libmerc/dns.h":
@@ -415,7 +415,7 @@ cdef class Mercury:
         cdef bytes dst_ip_b = dst_ip.encode()
         cdef char* dst_ip_c = dst_ip_b
 
-        cdef analysis_result ar = self.clf.perform_analysis(fp_str_c, server_name_c, dst_ip_c, dst_port, NULL)
+        cdef analysis_result ar = self.clf.perform_analysis(fp_str_c, server_name_c, dst_ip_c, dst_port, '')
 
         cdef fingerprint_status fp_status_enum = ar.status
         fp_status = fp_status_dict[fp_status_enum]

src/cython/mercury.pyx

@@ -33,7 +33,8 @@ classifier *analysis_init_from_archive(int, //verbosity
                                        enum enc_key_type key_type,
                                        const float fp_proc_threshold,
                                        const float proc_dst_threshold,
-                                       const bool report_os) {
+                                       const bool report_os,
+                                       const bool minimize_ram) {
 
     if (enc_key != NULL || key_type != enc_key_type_none) {
         //fprintf(stderr, "note: decryption key provided in configuration\n");
@@ -44,7 +45,7 @@ classifier *analysis_init_from_archive(int, //verbosity
     }
 
     encrypted_compressed_archive archive{archive_name, enc_key}; // TODO: key type
-    return new classifier(archive, fp_proc_threshold, proc_dst_threshold, report_os);
+    return new classifier(archive, fp_proc_threshold, proc_dst_threshold, report_os, minimize_ram);
 }
 
 

src/libmerc/analysis.cc

@@ -41,7 +41,8 @@ class classifier *analysis_init_from_archive(int verbosity,
                                enum enc_key_type key_type,
                                float fp_proc_threshold,
                                float proc_dst_threshold,
-                               bool report_os);
+                               bool report_os,
+                               bool minimize_ram=false);
 
 int analysis_finalize(classifier *c);
 
@@ -104,9 +105,10 @@ class fingerprint_data {
                      common_data *c,
                      bool &malware_database,
                      size_t total_cnt,
-                     bool report_os
+                     bool report_os,
+                     bool minimize_ram
                      ) :
-        classifier{process_info,total_cnt},
+        classifier{process_info, total_cnt, minimize_ram},
         malware_db{malware_database},
         subnet_data_ptr{subnets},
         common{c},
@@ -535,7 +537,7 @@ class classifier {
         return(true);
     }
 
-    void process_fp_db_line(std::string &line_str, bool report_os) {
+    void process_fp_db_line(std::string &line_str, bool report_os, bool minimize_ram) {
 
         rapidjson::Document fp;
         fp.Parse(line_str.c_str());
@@ -593,7 +595,8 @@ class classifier {
                                                              &common,
                                                              MALWARE_DB,
                                                              total_count,
-                                                             report_os
+                                                             report_os,
+                                                             minimize_ram
                                                              );
 
             if (fp.HasMember("str_repr") && fp["str_repr"].IsString()) {
@@ -655,7 +658,8 @@ class classifier {
     classifier(class encrypted_compressed_archive &archive,
                float fp_proc_threshold,
                float proc_dst_threshold,
-               bool report_os) : os_dictionary{}, subnets{}, fpdb{}, resource_version{} {
+               bool report_os,
+               bool minimize_ram) : os_dictionary{}, subnets{}, fpdb{}, resource_version{} {
 
         // reserve attribute for encrypted_dns watchlist
         //
@@ -710,7 +714,7 @@ class classifier {
                     if (threshold_set) {
                         printf_err(log_debug, "loading fingerprint_db_lite.json\n");
                         while (archive.getline(line_str)) {
-                            process_fp_db_line(line_str, report_os);
+                            process_fp_db_line(line_str, report_os, minimize_ram);
                         }
                         got_fp_db = true;
                         print_fp_counts();
@@ -723,7 +727,7 @@ class classifier {
                     else if (!threshold_set || lite_db || full_db) {
                         printf_err(log_debug, "loading fingerprint_db.json\n");
                         while (archive.getline(line_str)) {
-                            process_fp_db_line(line_str, report_os);
+                            process_fp_db_line(line_str, report_os, minimize_ram);
                         }
                         print_fp_counts();
                     }