- Code linting

- Update protobuf format

Author: kapot65

dfparser/__init__.py

@@ -1,21 +1,29 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Mon Dec 26 14:24:42 2016
+"""Init script.
 
-@author: chernov
+Set package version.
 """
-
 import os
 import sys
 
-cur_dir = os.path.dirname(os.path.realpath(__file__))
-if not cur_dir in sys.path: sys.path.append(cur_dir)
-del cur_dir
+from pkg_resources import get_distribution
 
-from envelope_parser import create_message, parse_from_file, parse_message
-from envelope_parser import read_machine_header, get_messages_from_stream
-from rsh_parser import RshPackage, serialise_to_rsh, parse_from_rsb
-from rsh_parser import serialize_to_rsb, dump_to_rsb
+CUR_DIR = os.path.dirname(os.path.realpath(__file__))
+if CUR_DIR not in sys.path:
+    sys.path.append(CUR_DIR)
+del CUR_DIR
+
+from df_data import def_values
+from df_data import type_codes
+from envelope_parser import create_message
+from envelope_parser import get_messages_from_stream
+from envelope_parser import parse_from_file
+from envelope_parser import parse_message
+from envelope_parser import read_machine_header
 from rsb_event_pb2 import Point
-from df_data import def_values, type_codes
+from rsh_parser import dump_to_rsb
+from rsh_parser import parse_from_rsb
+from rsh_parser import RshPackage
+from rsh_parser import serialise_to_rsh
+from rsh_parser import serialize_to_rsb
+
+__version__ = get_distribution('dfparser').version

dfparser/configs/rsb_event.proto

@@ -6,24 +6,27 @@ message Point {
   message Channel {
     message Block {
         // Необработанное событие
-        message Event {
-            uint64 time = 1; //время в наносекундах от начала блока
+        message Frame {
+            uint64 time = 1; // время первого бина в наносекундах
+                             // от начала блока
             bytes data = 2; // массив кадра события в формате int16
-                            // ед. измерения - каналы
+                            // ед. измерения - каналы, C-order
         }
         // Обработанные события. Содержат только время и амплитуду сигналов.
         // Для экономии места при сериализации амплитуды и времена лежат в
         // разных массивах. Амплитуда и время имеющие одинаковые индесы
         // соответствуют одному событию
-        message Peaks {
+        message Events {
             repeated uint64 times = 1; //время в наносекундах от начала блока
             repeated uint64 amplitudes = 2; //амплитуда события в каналах
         }
         uint64 time = 1; // время начала блока в наносекундах с начала эпохи
-        repeated Event events = 2; // массив необработанных событий
-        Peaks peaks = 3; // массив обработанных событий
+        repeated Frame frames = 2; // массив необработанных событий
+        Events events = 3; // массив обработанных событий
+        uint64 length = 4; // размер блока в наносекундах
+        uint64 bin_size = 5;  // размер бина в наносекундах
     }
-    uint64 num = 1; // номер канала
+    uint64 id = 1; // номер канала
     repeated Block blocks = 2; // набранные блоки
   }
   repeated Channel channels = 1; // массиив событий по каналам

dfparser/envelope_parser.py

@@ -1,52 +1,48 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Sep  9 20:54:46 2016
-
-@author: chernov
-"""
+"""Dataforge envelope format parser utils."""
 
+import json
 import os
-import sys
 import re
-import json
 import struct
+import sys
 import time
 
-cur_dir = os.path.dirname(os.path.realpath(__file__))
-if not cur_dir in sys.path: sys.path.append(cur_dir)
-del cur_dir
+CUR_DIR = os.path.dirname(os.path.realpath(__file__))
+if CUR_DIR not in sys.path:
+    sys.path.append(CUR_DIR)
+del CUR_DIR
+
+from df_data.type_codes import header_types
+from df_data.type_codes import meta_types
+
 
-from df_data.type_codes import meta_types, header_types
-        
 def create_message(json_meta: dict, data: bytearray=b'',
                    data_type: "binary_types"=0) -> bytearray:
-    """
-     Create message, containing meta and data in df-envelope format
-     @json_meta - metadata
-     @data - binary data
-     @data_type - data type code for binary data
-     @return - message as bytearray
-     
+    """Create message, containing meta and data in df-envelope format.
+
+    @json_meta - metadata
+    @data - binary data
+    @data_type - data type code for binary data
+    @return - message as bytearray
+
     """
     __check_data(data)
-    
+
     header = __create_machine_header(json_meta, data, data_type)
     meta = __prepare_meta(json_meta)
-    
+
     return header + meta + data
-    
-    
+
+
 def parse_from_file(filename: str, nodata: bool=False) \
-    -> [dict, dict, bytearray]:
-    """
-     Parse df message from file
-     @filename - path to file
-     @nodata - do not load data
-     @return - [binary header, metadata, binary data]
-     
+        -> [dict, dict, bytearray]:
+    """Parse df message from file.
+
+    @filename - path to file
+    @nodata - do not load data
+    @return - [binary header, metadata, binary data]
+
     """
-    
     header = None
     with open(filename, "rb") as file:
         header = read_machine_header(file.read(30))
@@ -55,19 +51,18 @@ def parse_from_file(filename: str, nodata: bool=False) \
         data = b''
         if not nodata:
             data = file.read(header['data_len'])
-        return header, meta, data 
-    
-    
+        return header, meta, data
+
+
 def parse_message(message: bytearray, nodata: bool=False) \
-    -> [dict, dict, bytearray]:
-    """
-     Parse df message from bytearray
-     @message - message data
-     @nodata - do not load data
-     @return - [binary header, metadata, binary data]
-     
+        -> [dict, dict, bytearray]:
+    """Parse df message from bytearray.
+
+    @message - message data
+    @nodata - do not load data
+    @return - [binary header, metadata, binary data]
+
     """
-    
     header = read_machine_header(message)
     meta_raw = message[30:30 + header['meta_len']]
     meta = __parse_meta(meta_raw, header)
@@ -77,32 +72,32 @@ def parse_message(message: bytearray, nodata: bool=False) \
         data = message[data_start:data_start + header['data_len']]
     return header, meta, data
 
-    
+
 def read_machine_header(data: bytearray) -> dict:
+    """Parse binary header.
+
+    @data - bytearray, contains binary header
+    @return - parsed binary header
+
     """
-     Parse binary header
-     @data - bytearray, contains binary header
-     @return - parsed binary header
-    """
-    
     header = dict()
     header['type'] = struct.unpack('>I', data[2:6])[0]
     header['time'] = struct.unpack('>I', data[6:10])[0]
     header['meta_type'] = struct.unpack('>I', data[10:14])[0]
     header['meta_len'] = struct.unpack('>I', data[14:18])[0]
     header['data_type'] = struct.unpack('>I', data[18:22])[0]
     header['data_len'] = struct.unpack('>I', data[22:26])[0]
-    
+
     return header
-    
-    
+
+
 def get_messages_from_stream(data: bytearray) \
-    -> [[{dict, dict, bytearray}], bytearray]:
-    """
-      Extract complete messages from stream and cut out them from stream
-      @data - stream binary data
-      @return - [list of messages, choped stream data]
-      
+        -> [[{dict, dict, bytearray}], bytearray]:
+    """Extract complete messages from stream and cut out them from stream.
+
+    @data - stream binary data
+    @return - [list of messages, choped stream data]
+
     """
     messages = []
     iterator = get_messages_from_stream.header_re.finditer(data)
@@ -115,63 +110,63 @@ def get_messages_from_stream(data: bytearray) \
 
         if cur_last_pos > len(data):
             break
-        
+
         header, meta, bin_data = parse_message(data[pos:])
         messages.append({'header': header, 'meta': meta, 'data': bin_data})
-        
+
         last_pos = cur_last_pos
-        
+
     data = data[last_pos:]
     return messages, data
-        
+
 get_messages_from_stream.header_re = re.compile(b"#!.{24}!#", re.DOTALL)
-    
+
 
 def __parse_meta(meta_raw, header):
     if header["meta_type"] == meta_types["JSON_METATYPE"]:
         return json.loads(meta_raw.decode())
     else:
-        err = "Parsing meta type %s not implemented"%(bin(header["meta_type"]))
+        err = "Parsing meta type %s not implemented" % \
+            (bin(header["meta_type"]))
         raise NotImplementedError(err)
-        
-    
+
+
 def __prepare_meta(json_meta):
-    if type(json_meta) is dict:
+    if isinstance(json_meta, dict):
         json_meta = json.dumps(json_meta, indent=4).encode()
         json_meta += b'\r\n\r\n'
-    elif not type(json_meta) is str:
+    elif isinstance(json_meta, str):
         raise ValueError("Input meta should be dict or str")
     return json_meta
 
 
 def __check_data(data):
-    if not type(data) is bytes:
+    if isinstance(data, bytes):
         raise ValueError("Input data should have bytes type")
-        
+
 
 def __create_machine_header(json_meta: dict, data: bytearray=b'',
                             data_type: "binary_types"=0) -> bytearray:
-    
+
     json_meta = __prepare_meta(json_meta)
     __check_data(data)
-    
+
     binary_header = b'#!'
-    
-    #binary header type
+
+    # binary header type
     binary_header += struct.pack('>I', header_types["DEFAULT"])
     millis = int(round(time.time() * 1000))
-    #current time
+    # current time
     binary_header += struct.pack('>Q', millis)[4:]
-    #meta type
+    # meta type
     binary_header += struct.pack('>I', meta_types["JSON_METATYPE"])
-    #meta length
+    # meta length
     binary_header += struct.pack('>I', len(json_meta))
-    #data type
+    # data type
     binary_header += struct.pack('>I', data_type)
-    #data length
+    # data length
     binary_header += struct.pack('>I', len(data))
-    
+
     binary_header += b'!#\r\n'
-    
-    return binary_header
 
+    return binary_header