sdlog2_dump.py now generates customizible CSV output, messages/fields filter added.

1 files changed, 159 insertions, 46 deletions

diff --git a/Tools/sdlog2_dump.py b/Tools/sdlog2_dump.py
index b74bf4739..c8fc00a59 100644
--- a/Tools/sdlog2_dump.py
+++ b/Tools/sdlog2_dump.py
@@ -1,11 +1,21 @@
 #!/usr/bin/env python
 
-"""Parse and dump binary log generated by sdlog2
+"""Dump binary log generated by sdlog2 or APM as CSV
     
-    Usage: python sdlog2_dump.py <log.bin>"""
+Usage: python sdlog2_dump.py <log.bin> [-v] [-d delimiter] [-n null] [-m MSG[.field1,field2,...]]
+    
+    -v  Use plain debug output instead of CSV.
+    
+    -d  Use "delimiter" in CSV. Default is ",".
+    
+    -n  Use "null" as placeholder for empty values in CSV. Default is empty.
+    
+    -m MSG[.field1,field2,...]
+        Dump only messages of specified type, and only specified fields.
+        Multiple -m options allowed."""
 
 __author__  = "Anton Babushkin"
-__version__ = "0.2"
+__version__ = "1.0"
 
 import struct, sys
 
@@ -40,54 +50,83 @@ class SDLog2Parser:
         "q": ("q", None),
         "Q": ("Q", None),
     }
+    __csv_delim = ","
+    __csv_null = ""
+    __msg_filter = {}
+    __debug_out = False
     
     def __init__(self):
         return
 
     def reset(self):
-        self.msg_descrs = {}
-        self.buffer = ""
-        self.ptr = 0
+        self.__msg_descrs = {}
+        self.__buffer = ""
+        self.__ptr = 0
+        self.__csv_columns = []
+        self.__csv_data = {}
+    
+    def setCSVDelimiter(self, csv_delim):
+        self.__csv_delim = csv_delim
+    
+    def setCSVNull(self, csv_null):
+        self.__csv_null = csv_null
+    
+    def setMsgFilter(self, msg_filter):
+        self.__msg_filter = msg_filter
+    
+    def setDebugOut(self, debug_out):
+        self.__debug_out = debug_out
     
     def process(self, fn):
         self.reset()
+        first_data_msg = True
         f = open(fn, "r")
         while True:
             chunk = f.read(self.BLOCK_SIZE)
             if len(chunk) == 0:
                 break
-            self.buffer = self.buffer[self.ptr:] + chunk
-            self.ptr = 0
-            while self._bytes_left() >= self.MSG_HEADER_LEN:
-                head1 = ord(self.buffer[self.ptr])
-                head2 = ord(self.buffer[self.ptr+1])
+            self.__buffer = self.__buffer[self.__ptr:] + chunk
+            self.__ptr = 0
+            while self.__bytesLeft() >= self.MSG_HEADER_LEN:
+                head1 = ord(self.__buffer[self.__ptr])
+                head2 = ord(self.__buffer[self.__ptr+1])
                 if (head1 != self.MSG_HEAD1 or head2 != self.MSG_HEAD2):
                     raise Exception("Invalid header: %02X %02X, must be %02X %02X" % (head1, head2, self.MSG_HEAD1, self.MSG_HEAD2))
-                msg_type = ord(self.buffer[self.ptr+2])
+                msg_type = ord(self.__buffer[self.__ptr+2])
                 if msg_type == self.MSG_TYPE_FORMAT:
-                    self._parse_msg_descr()
+                    self.__parseMsgDescr()
                 else:
-                    msg_descr = self.msg_descrs[msg_type]
+                    msg_descr = self.__msg_descrs[msg_type]
                     if msg_descr == None:
                         raise Exception("Unknown msg type: %i" % msg_type)
                     msg_length = msg_descr[0]
-                    if self._bytes_left() < msg_length:
+                    if self.__bytesLeft() < msg_length:
                         break
-                    self._parse_msg(msg_descr)
+                    if first_data_msg:
+                        print self.__csv_delim.join(self.__csv_columns)
+                        first_data_msg = False
+                    self.__parseMsg(msg_descr)
+    
         f.close()
 
-    def _bytes_left(self):
-        return len(self.buffer) - self.ptr
+    def __bytesLeft(self):
+        return len(self.__buffer) - self.__ptr
 
-    def _parse_msg_descr(self):
-        if self._bytes_left() < self.MSG_FORMAT_PACKET_LEN:
-            raise BufferUnderflow("Data is too short: %i bytes, need %i" % (self._bytes_left(), self.MSG_FORMAT_PACKET_LEN))
-        data = struct.unpack(self.MSG_FORMAT_STRUCT, self.buffer[self.ptr + 3 : self.ptr + self.MSG_FORMAT_PACKET_LEN])
+    def __filterMsg(self, msg_name):
+        show_fields = "*"
+        if len(self.__msg_filter) > 0:
+            show_fields = self.__msg_filter.get(msg_name)
+        return show_fields
+
+    def __parseMsgDescr(self):
+        if self.__bytesLeft() < self.MSG_FORMAT_PACKET_LEN:
+            raise BufferUnderflow("Data is too short: %i bytes, need %i" % (self.__bytesLeft(), self.MSG_FORMAT_PACKET_LEN))
+        data = struct.unpack(self.MSG_FORMAT_STRUCT, self.__buffer[self.__ptr + 3 : self.__ptr + self.MSG_FORMAT_PACKET_LEN])
         msg_type = data[0]
         msg_length = data[1]
-        msg_name = data[2].strip('\0')
-        msg_format = data[3].strip('\0')
-        msg_labels = data[4].strip('\0').split(",")
+        msg_name = data[2].strip("\0")
+        msg_format = data[3].strip("\0")
+        msg_labels = data[4].strip("\0").split(",")
         # Convert msg_format to struct.unpack format string
         msg_struct = ""
         msg_mults = []
@@ -97,34 +136,108 @@ class SDLog2Parser:
                 msg_struct += f[0]
                 msg_mults.append(f[1])
             except KeyError as e:
-                raise Exception("Unsupported format char: %s in message %s (0x%02X)" % (c, msg_name, msg_type))
-        msg_struct = "<" + msg_struct
-        print msg_format, msg_struct
-        print "MSG FORMAT: type = %i, length = %i, name = %s, format = %s, labels = %s, struct = %s, mults = %s" % (msg_type, msg_length, msg_name, msg_format, str(msg_labels), msg_struct, msg_mults)
-        self.msg_descrs[msg_type] = (msg_length, msg_name, msg_format, msg_labels, msg_struct, msg_mults)
-        self.ptr += self.MSG_FORMAT_PACKET_LEN
+                raise Exception("Unsupported format char: %s in message %s (%i)" % (c, msg_name, msg_type))
+        msg_struct = "<" + msg_struct   # force little-endian
+        self.__msg_descrs[msg_type] = (msg_length, msg_name, msg_format, msg_labels, msg_struct, msg_mults)
+        show_fields = self.__filterMsg(msg_name)
+        if show_fields != None:
+            if self.__debug_out:
+                print "MSG FORMAT: type = %i, length = %i, name = %s, format = %s, labels = %s, struct = %s, mults = %s" % (
+                            msg_type, msg_length, msg_name, msg_format, str(msg_labels), msg_struct, msg_mults)
+            else:
+                if show_fields == "*":
+                    fields = msg_labels
+                else:
+                    fields = []
+                    for field in show_fields:
+                        if field in msg_labels:
+                            fields.append(field)
+                for field in fields:
+                    msg_field = msg_name + "." + field
+                    self.__csv_columns.append(msg_field)
+                    self.__csv_data[msg_field] = None
+        self.__ptr += self.MSG_FORMAT_PACKET_LEN
 
-    def _parse_msg(self, msg_descr):
+    def __parseMsg(self, msg_descr):
         msg_length, msg_name, msg_format, msg_labels, msg_struct, msg_mults = msg_descr
-        data = list(struct.unpack(msg_struct, self.buffer[self.ptr+self.MSG_HEADER_LEN:self.ptr+msg_length]))
-        s = []
-        for i in xrange(len(data)):
-            if type(data[i]) is str:
-                data[i] = data[i].strip('\0')
-            m = msg_mults[i]
-            if m != None:
-                data[i] = data[i] * m
-            s.append(msg_labels[i] + "=" + str(data[i]))
-        
-        print "MSG %s: %s" % (msg_name, ", ".join(s))
-        self.ptr += msg_length
-    
+        show_fields = self.__filterMsg(msg_name)
+        if (show_fields != None):
+            data = list(struct.unpack(msg_struct, self.__buffer[self.__ptr+self.MSG_HEADER_LEN:self.__ptr+msg_length]))
+            for i in xrange(len(data)):
+                if type(data[i]) is str:
+                    data[i] = data[i].strip("\0")
+                m = msg_mults[i]
+                if m != None:
+                    data[i] = data[i] * m
+            if self.__debug_out:
+                s = []
+                for i in xrange(len(data)):
+                    label = msg_labels[i]
+                    if show_fields == "*" or label in show_fields:
+                        s.append(label + "=" + str(data[i]))
+                print "MSG %s: %s" % (msg_name, ", ".join(s))
+            else:
+                # update CSV data buffer
+                for i in xrange(len(data)):
+                    label = msg_labels[i]
+                    if show_fields == "*" or label in show_fields:
+                        self.__csv_data[msg_name + "." + label] = data[i]
+                # format and print CSV row
+                s = []
+                for field in self.__csv_columns:
+                    v = self.__csv_data[field]
+                    if v == None:
+                        v = self.__csv_null
+                    else:
+                        v = str(v)
+                    s.append(v)
+                print self.__csv_delim.join(s)
+        self.__ptr += msg_length
+
 def _main():
     if len(sys.argv) < 2:
-        print "Usage:\npython sdlog2_dump.py <log.bin>"
+        print "Usage: python sdlog2_dump.py <log.bin> [-v] [-d delimiter] [-n null] [-m MSG[.field1,field2,...]]\n"
+        print "\t-v\tUse plain debug output instead of CSV.\n"
+        print "\t-d\tUse \"delimiter\" in CSV. Default is \",\".\n"
+        print "\t-n\tUse \"null\" as placeholder for empty values in CSV. Default is empty.\n"
+        print "\t-m MSG[.field1,field2,...]\n\t\tDump only messages of specified type, and only specified fields.\n\t\tMultiple -m options allowed."
         return
     fn = sys.argv[1]
+    debug_out = False
+    msg_filter = {}
+    csv_null = ""
+    csv_delim = ","
+    opt = None
+    for arg in sys.argv[2:]:
+        if opt != None:
+            if opt == "d":
+                csv_delim = arg
+            elif opt == "n":
+                csv_null = arg
+            if opt == "m":
+                show_fields = "*"
+                a = arg.split(".")
+                if len(a) > 1:
+                    show_fields = set(a[1].split(","))
+                msg_filter[a[0]] = show_fields
+            opt = None
+        else:
+            if arg == "-v":
+                debug_out = True
+            elif arg == "-d":
+                opt = "d"
+            elif arg == "-n":
+                opt = "n"
+            elif arg == "-m":
+                opt = "m"
+
+    if csv_delim == "\\t":
+        csv_delim = "\t"
     parser = SDLog2Parser()
+    parser.setCSVDelimiter(csv_delim)
+    parser.setCSVNull(csv_null)
+    parser.setMsgFilter(msg_filter)
+    parser.setDebugOut(debug_out)
     parser.process(fn)
 
 if __name__ == "__main__":