chore: mirror serial2pcap in case it goes away

Author: timyardley

tools/mirrored/serial2pcap/DISCLAIMER.md

@@ -0,0 +1,9 @@
+## Disclaimer of Warranty
+This Work is provided "as is". Any express or implied warranties, including but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. In no event shall the United States Government be liable for any direct, indirect, incidental, special, exemplary or consequential damages (including, but not limited to, procurement of substitute goods or services, loss of use, data or profits, or business interruption) however caused and on any theory of liability, whether in contract, strict liability, or tort (including negligence or otherwise) arising in any way out of the use of this Work, even if advised of the possibility of such damage.
+
+The User of this Work agrees to hold harmless and indemnify the United States Government, its agents and employees from every claim or liability (whether in tort or in contract), including attorneys' fees, court costs, and expenses, arising in direct consequence of Recipient's use of the item, including but not limited to, claims or liabilities made for injury to or death of personnel of User or third parties, damage to or destruction of property of User or third parties, infringement or other violations of intellectual property or technical data rights.
+
+Nothing in this Work is intended to constitute an endorsement, explicit or implied, by the United States Government of any particular manufacturer's product or service.
+
+## Disclaimer of Endorsement
+Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or otherwise, in this Work does not constitute an endorsement, recommendation, or favoring by the United States Government and shall not be used for advertising or product endorsement purposes.

tools/mirrored/serial2pcap/HASHES

@@ -0,0 +1,23 @@
+		MD5				SHA-1
+-------------------------------------------------------------------------
+64ece21dd9b0da617238fcd98c6b344d 24e2f0a0becabb3dfaf8ea88614d898f76dafc2f serial2pcap-release\Lib\BlankConsumer.py
+fe3544e4a53fd20bbb0524d9b92abebe 4233c9efec69710bbdbfb7aba300792f28f61fe9 serial2pcap-release\Lib\BufferOutput.py
+c14022bc04c8f5a1351f14115a9afc0e 1947a0376c2ab1e29a2a61ab35ba35153d173740 serial2pcap-release\Lib\CaptureDataBuffer.py
+337e4d83c00b2033583205f03086d560 db3778238c777bdebf7beb672a0f9cbe1d80cff2 serial2pcap-release\Lib\DumpTimeFile.py
+213b491b7500e9c8319246468dfbfc40 1f6f6dd57f52d542d96ac4f24168595a99de48df serial2pcap-release\Lib\SerialPortReader.py
+67a6211a8d90e98865e949206b85c2e7 8076566896c949c800e0772de163dc1d86344961 serial2pcap-release\Lib\util.py
+d41d8cd98f00b204e9800998ecf8427e da39a3ee5e6b4b0d3255bfef95601890afd80709 serial2pcap-release\Lib\__init__.py
+183e0af194aedc8721d5d3eb8a1c896b 8a980c7c8fae69a5345e5b7c10d42432701436e6 serial2pcap-release\libserial2pcap.py
+b90fbc29ace5fdcea1d4b2a1d8807397 c2582cc0d2a5c50db3c132342c088386415f0131 serial2pcap-release\libserial2pcapunittest.py
+c994c906fc62bd2bffe342b01935cfc6 48d58a55d5ed3518441667cd0be5c76089d9e284 serial2pcap-release\Plugins\Byte.py
+37ffd33b191ebecb6a9952a4ddf470a4 dc5b0762171a4376b7d641d8e99320d57e507109 serial2pcap-release\Plugins\df1.py
+58f6f0d2ab98e348b01d72a1baae8209 fd82bbf331b9a0a40f2ae97fe002a352e0428ea2 serial2pcap-release\Plugins\DNP3.py
+e850bed0ea08debf8a6ef46dd0b23d9c 8250ebe4eabdb5001f657b4c5c929d47cafe2198 serial2pcap-release\Plugins\ModbusASCII.py
+d99c88249f2df4150525a37c6c8be03a 481fede6ba631136c4d7631bb44dcd8db2f33591 serial2pcap-release\Plugins\ModbusRTU.py
+f13f39d70ae5faaba7eebb9b292defed 6d2f406cd0c6277d13bf19674949a7c05eb4ab56 serial2pcap-release\Plugins\PluginCore.py
+a2efadba5e741d2f12481e0fb25ecbcb 33721025d6787a3ea743a8b6501e7640a4200977 serial2pcap-release\Plugins\SELFM.py
+4b53bdbb95102b2240a8bfde16a24c55 ea2f6c02fbad6d493931653c02064b75014b83fe serial2pcap-release\Plugins\SLIP.py
+d41d8cd98f00b204e9800998ecf8427e da39a3ee5e6b4b0d3255bfef95601890afd80709 serial2pcap-release\Plugins\__init__.py
+1c136b47f801afbfef6f00c91a4e20a9 5b84aba38967e3b2a9eb8f1e0b870e050628c094 serial2pcap-release\serial2pcap.py
+b5fefb987fded860ac6650e59a944143 9e013796c25fae0be2078676c23defcefc893dd6 serial2pcap-release\USAGE
+3cf918272ffa5de195752d73f3da3e5e 7959c969e092f2a5a8604e2287807ac5b1b384ad serial2pcap-release\VERSION

tools/mirrored/serial2pcap/LICENSE.md

@@ -0,0 +1,3 @@
+This Work was prepared by a United States Government employee and, therefore, is excluded from copyright by Section 105 of the Copyright Act of 1976.
+
+Copyright and Related Rights in the Work worldwide are waived through the [CC0 1.0](https://creativecommons.org/publicdomain/zero/1.0/) [Universal license](https://creativecommons.org/publicdomain/zero/1.0/legalcode).

tools/mirrored/serial2pcap/Lib/BlankConsumer.py

@@ -0,0 +1,19 @@
+#!/usr/bin/python
+
+import time
+import os
+import signal
+
+def BlankConsumer(datastorage, stopevent):
+    while not stopevent.isSet():
+        if len(datastorage.raw) > 1:
+            datastorage.pop_front(1)
+        else:
+            datastorage.newdata.clear()
+            #let other threads exit
+            time.sleep(1)
+            #simulate a ctrl-c event so that the higher level thread will exit
+            os.kill(os.getpid(), signal.SIGINT)
+            #need to break so that this thread doesn't try to catch the keyboard interrupt
+            break
+    

tools/mirrored/serial2pcap/Lib/BufferOutput.py

@@ -0,0 +1,243 @@
+#!/usr/bin/python
+
+import time
+import struct
+import logging
+import traceback
+
+from CaptureDataBuffer import FileNameTracking
+
+def MakePCAPHeader(filehandle, datalinktype):
+    magic_number = struct.pack("I", int(0xa1b2c3d4))
+    version_major = struct.pack("H", 2)
+    version_minor = struct.pack("H", 4)
+    thiszone = struct.pack("I", 0)
+    sigfigs = struct.pack("I", 0)
+    snaplen = struct.pack("I", 65535)
+    network = struct.pack("I", datalinktype)
+    
+    filehandle.write(magic_number)
+    filehandle.write(version_major)
+    filehandle.write(version_minor)
+    filehandle.write(thiszone)
+    filehandle.write(sigfigs)
+    filehandle.write(snaplen)
+    filehandle.write(network)
+    
+    filehandle.flush()
+    
+#This function takes some data and creates a packet entry within a PCAP file.
+#filehandle is a file handle to an open PCAP file
+#data is the data to be written
+#CallBackFunction is a function that can intercept the data and return data to be added to the packet - it can also modify the data if it wants
+def MakePCAPEntry(filehandle, data, CallBackFunction):
+    databuffer = ""
+    
+    for byte in data:
+        databuffer += byte
+    
+    #retrieve the capture time from the last byte in the array
+    ftime = data[len(data)-1].GetTime()
+    
+    #do timing calculations to separate seconds from ms
+    seconds = int(ftime)
+    mseconds = int((ftime - int(ftime)) * 1000000)
+    
+    #run the callback function - this allows programs to intercept the output before its written to disk
+    callbackdata = CallBackFunction(seconds, mseconds, databuffer, None)
+    
+    #Build the packet header within the PCAP file - this includes the time stamp and length
+    ts_sec = struct.pack("I", seconds)
+    ts_usec = struct.pack("I", mseconds)
+    incl_len = struct.pack("I", len(data) + len(callbackdata))
+    orig_len = struct.pack("I", len(data) + len(callbackdata))
+    
+    #write the packet header to disk
+    filehandle.write(ts_sec)
+    filehandle.write(ts_usec)
+    filehandle.write(incl_len)
+    filehandle.write(orig_len)
+    
+    #if there is any callback data then write it to disk
+    if callbackdata is not None:
+        filehandle.write(callbackdata)
+    
+    #finally write the actual data to disk
+    filehandle.write(databuffer)
+    
+    #flush the data to make sure it hits disk
+    filehandle.flush()
+    
+#this function is used when used with libserial2pcap - it is responsible for taking identified packets from BufferOutput and storing them in a shared PCAP buffer
+#pcapbuffer - internal buffer used to store processed packets and header information
+#data - actual packet data that will be stored in the pcapbuffer
+#CallBackFunction - function pointer from a processing plugin that needs to be called before storing data
+#datalinktype - the data link type as provided from the processing plugin
+def AddToPCAPBuffer(pcapbuffer, data, CallBackFunction, datalinktype):
+    databuffer = ""
+    
+    for byte in data:
+        databuffer += byte
+    
+    #retrieve the capture time from the last byte in the array
+    ftime = data[len(data)-1].GetTime()
+    
+    #do timing calculations to separate seconds from ms
+    seconds = int(ftime)
+    mseconds = int((ftime - int(ftime)) * 1000000)
+    
+    #run the callback function - this allows programs to intercept the output before its written to disk
+    callbackdata = CallBackFunction(seconds, mseconds, databuffer, None)
+    
+    #create the packetdata structure
+    packetdata = {}
+
+    packetdata["timestamp_s"] = seconds
+    packetdata["timestamp_ms"] = mseconds
+    packetdata["length"] = len(callbackdata)
+    packetdata["dlt"] = datalinktype
+
+    pcapbuffer.lock.acquire()
+    pcapbuffer.raw.append({"pkthdr": packetdata, "packet": callbackdata + databuffer})
+    pcapbuffer.lock.release()
+    
+def BufferOutput(datastorage, outfilename, plugin, stopevent, debug, pcapbuffer):
+    logger = logging.getLogger("serial2pcap").getChild(__name__)
+    if debug:
+        logger.setLevel(logging.DEBUG)
+    
+    #this should check to make sure that either outfilename or (xor) pcap buffer are used
+    #logic further down relies on one or the other being set
+    if outfilename is not None and pcapbuffer is not None:
+        raise RuntimeError("Can Not Use Output File And PCAP Buffer Together")
+    elif outfilename is None and pcapbuffer is None:
+        raise RuntimeError("Must Use Either Output File Or PCAP Buffer")
+    
+    fnt = FileNameTracking()
+    
+    unknownbuffer = []
+    
+    logger.info("Starting to Process Data")
+    
+    #this only needs to be done if using an outfile
+    if outfilename is not None:
+        try:
+            filehandle = open(outfilename, "wb")
+        except IOError:
+            logger.error("Could Not Open Output File")
+            stopevent.set()
+            return
+        MakePCAPHeader(filehandle, plugin.PluginDataLinkType)
+    
+    while not stopevent.isSet():
+        if datastorage.newdata.isSet():
+            datastorage.lock.acquire()
+    
+            try:
+                (pluginresult,packetlength) = plugin.Identify(datastorage.raw, datastorage.capture_info)
+            except Exception as e:
+                logger.error("Caught Exception in the plugin, Shutting Down")
+                logger.error("Exception Was: " + traceback.format_exc())
+                stopevent.set()
+                datastorage.lock.release()
+                return
+            
+            if pluginresult == plugin.Status.UNKNOWN:
+                logger.debug("Got Status.UNKNOWN, data was: " + datastorage.raw[0])
+                unknownbuffer.append(datastorage.raw[0])
+                datastorage.pop_front(1)
+            
+            elif pluginresult == plugin.Status.OK or pluginresult == plugin.Status.INVALID:
+                #if packet length is 0 then this is probably an error. Treat it the same as an UNKNOWN event otherwise a deadlock may occur				
+                if packetlength <= 0:
+                    logger.debug("Got Status.OK | Status.INVALID, but the packet length was <= 0. Treating as Status.UNKNOWN")
+                    unknownbuffer.append(datastorage.raw[0])
+                    datastorage.pop_front(1)
+    
+                #otherwise process it as normal
+                else:
+                    if len(unknownbuffer) > 0:
+                        logger.debug("Flushing Unknown Buffer To PCAP:")
+                        logger.debug(unknownbuffer)
+    
+                        #this if/else checks to see if output needs to go to a file or the pcapbuffer
+                        if outfilename is not None:
+                            #check to see if the output file needs to be split
+                            if datastorage.DoSplit(filehandle) == True:
+                                #if so then close the original file
+                                filehandle.close()
+                                #get a new file handle with the new naming convention
+                                filehandle = fnt.GetNewFileHandle(outfilename)
+                                #make sure it gets the pcap header
+                                MakePCAPHeader(filehandle, plugin.PluginDataLinkType)
+    
+                            MakePCAPEntry(filehandle, unknownbuffer, plugin.OutputCallback)
+    
+                        else:
+                            #adds the unknown buffer to the pcap buffer
+                            AddToPCAPBuffer(pcapbuffer, unknownbuffer, plugin.OutputCallback, plugin.PluginDataLinkType)
+    
+                        datastorage.inc_packet_counter()
+    
+                        unknownbuffer = []
+                    
+                    logger.debug("Got Status.OK | Status.INVALID, Packet Length: " + str(packetlength) + " flushing packet to PCAP")
+                    logger.debug(datastorage.raw[:packetlength])
+    
+                    #this if/else checks to see if output needs to go to a file or the pcapbuffer
+                    if outfilename is not None:
+                        #check to see if the output file needs to be split
+                        if datastorage.DoSplit(filehandle) == True:
+                            #if so then close the original file
+                            filehandle.close()
+                            #get a new file handle with the new naming convention
+                            filehandle = fnt.GetNewFileHandle(outfilename)
+                            #make sure it gets the pcap header
+                            MakePCAPHeader(filehandle, plugin.PluginDataLinkType)
+    
+                        MakePCAPEntry(filehandle, datastorage.raw[:packetlength], plugin.OutputCallback)
+    
+                    else:
+                        #adds the identified packet to the pcap buffer
+                        AddToPCAPBuffer(pcapbuffer, datastorage.raw[:packetlength], plugin.OutputCallback, plugin.PluginDataLinkType)
+    
+                    datastorage.inc_packet_counter()
+    
+                    datastorage.pop_front(packetlength)
+            
+            elif pluginresult == plugin.Status.TOOSHORT:
+                logger.debug("Got Status.TOOSHORT")
+                datastorage.newdata.clear()
+                
+            else:
+                logger.warning("Unknown Response Code From Plugin - This Will Be Ignored") 
+            
+            if len(datastorage.raw) == 0:
+                datastorage.newdata.clear()
+    
+            datastorage.lock.release()
+    
+        else:
+            time.sleep(.1)
+    
+    #end of while loop
+    
+    #if there is any data still left after the main while loop then move it over to the unknown buffer
+    for i in range(0,len(datastorage.raw)):
+        unknownbuffer.append(datastorage.raw[i])
+    
+    #dump anything in the unknown buffer to a single packet
+    if len(unknownbuffer) > 0:
+        logger.debug("Flushing Unknown Buffer To PCAP:")
+        logger.debug(unknownbuffer)
+        #this if/else checks to see if output needs to go to a file or the pcapbuffer
+        if outfilename is not None:
+            #for now not splitting files at this code since the code writes one big packet and there is not fine grained control over the size
+            MakePCAPEntry(filehandle, unknownbuffer, plugin.OutputCallback)
+        else:
+            #adds the unknown buffer to the pcap buffer
+            AddToPCAPBuffer(pcapbuffer, unknownbuffer, plugin.OutputCallback, plugin.PluginDataLinkType)
+        datastorage.inc_packet_counter()
+        logger.info("Done Processing")
+        
+    

tools/mirrored/serial2pcap/Lib/CaptureDataBuffer.py

@@ -0,0 +1,78 @@
+#!/usr/bin/python
+
+import threading
+import time
+
+class FileNameTracking:
+    def __init__(self):
+        pass
+
+    filecount = 0
+    
+    def GetNewFileHandle(self, OriginalFileName, mode="wb"):
+        self.filecount += 1
+    
+        return open(OriginalFileName + "." + str(self.filecount), mode)
+        
+    
+    
+class CaptureInformation:
+    def __init__(self):
+        pass
+
+    backing = None
+    baudrate = None
+    bytesize = None
+    parity = None
+    stopbits = None
+    
+class CaptureDataBuffer:
+    lock = None
+    newdata = None
+    raw = None
+    
+    total_packets_captured = 0
+    total_bytes_captured = 0
+    start_time = 0.0
+    file_split_size = 0
+    
+    capture_info = None
+    
+    def __init__(self):
+        self.lock = threading.Lock()
+        self.newdata = threading.Event()
+        self.raw = []
+    
+        self.capture_info = CaptureInformation()
+    
+        self.start_time = time.time()
+    
+    def pop_front(self, count):
+        if count >= len(self.raw):
+            self.raw = []
+    
+        else:
+            for i in range(0, count):
+                self.raw.pop(0)
+    
+    def inc_packet_counter(self, count=1):
+        self.total_packets_captured += count
+    
+    def inc_bytes_counter(self, count=1):
+        self.total_bytes_captured += count
+    
+    def DoSplit(self, filehandle):
+        if self.file_split_size != 0 and filehandle.tell() >= self.file_split_size:
+            return True
+        else:
+            return False
+    
+class CaptureDataBufferSimple:
+    lock = None
+    raw = None
+    
+    def __init__(self):
+        self.lock = threading.Lock()
+        self.raw = []
+    
+