diff options
Diffstat (limited to 'Tools')
-rw-r--r-- | Tools/logconv.py | 59 | ||||
-rw-r--r-- | Tools/sdlog2_dump.py | 293 |
2 files changed, 352 insertions, 0 deletions
diff --git a/Tools/logconv.py b/Tools/logconv.py new file mode 100644 index 000000000..c47d22a45 --- /dev/null +++ b/Tools/logconv.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +"""Convert binary log generated by sdlog to CSV format + +Usage: python logconv.py <log.bin>""" + +__author__ = "Anton Babushkin" +__version__ = "0.1" + +import struct, sys + +def _unpack_packet(data): + s = "" + s += "Q" #.timestamp = buf.raw.timestamp, + s += "fff" #.gyro = {buf.raw.gyro_rad_s[0], buf.raw.gyro_rad_s[1], buf.raw.gyro_rad_s[2]}, + s += "fff" #.accel = {buf.raw.accelerometer_m_s2[0], buf.raw.accelerometer_m_s2[1], buf.raw.accelerometer_m_s2[2]}, + s += "fff" #.mag = {buf.raw.magnetometer_ga[0], buf.raw.magnetometer_ga[1], buf.raw.magnetometer_ga[2]}, + s += "f" #.baro = buf.raw.baro_pres_mbar, + s += "f" #.baro_alt = buf.raw.baro_alt_meter, + s += "f" #.baro_temp = buf.raw.baro_temp_celcius, + s += "ffff" #.control = {buf.act_controls.control[0], buf.act_controls.control[1], buf.act_controls.control[2], buf.act_controls.control[3]}, + s += "ffffffff" #.actuators = {buf.act_outputs.output[0], buf.act_outputs.output[1], buf.act_outputs.output[2], buf.act_outputs.output[3], buf.act_outputs.output[4], buf.act_outputs.output[5], buf.act_outputs.output[6], buf.act_outputs.output[7]}, + s += "f" #.vbat = buf.batt.voltage_v, + s += "f" #.bat_current = buf.batt.current_a, + s += "f" #.bat_discharged = buf.batt.discharged_mah, + s += "ffff" #.adc = {buf.raw.adc_voltage_v[0], buf.raw.adc_voltage_v[1], buf.raw.adc_voltage_v[2], buf.raw.adc_voltage_v[3]}, + s += "fff" #.local_position = {buf.local_pos.x, buf.local_pos.y, buf.local_pos.z}, + s += "iii" #.gps_raw_position = {buf.gps_pos.lat, buf.gps_pos.lon, buf.gps_pos.alt}, + s += "fff" #.attitude = {buf.att.pitch, buf.att.roll, buf.att.yaw}, + s += "fffffffff" #.rotMatrix = {buf.att.R[0][0], buf.att.R[0][1], buf.att.R[0][2], buf.att.R[1][0], buf.att.R[1][1], buf.att.R[1][2], buf.att.R[2][0], buf.att.R[2][1], buf.att.R[2][2]}, + s += "fff" #.vicon = {buf.vicon_pos.x, buf.vicon_pos.y, buf.vicon_pos.z, buf.vicon_pos.roll, buf.vicon_pos.pitch, buf.vicon_pos.yaw}, + s += "ffff" #.control_effective = {buf.act_controls_effective.control_effective[0], buf.act_controls_effective.control_effective[1], buf.act_controls_effective.control_effective[2], buf.act_controls_effective.control_effective[3]}, + s += "ffffff" #.flow = {buf.flow.flow_raw_x, buf.flow.flow_raw_y, buf.flow.flow_comp_x_m, buf.flow.flow_comp_y_m, buf.flow.ground_distance_m, buf.flow.quality}, + s += "f" #.diff_pressure = buf.diff_pres.differential_pressure_pa, + s += "f" #.ind_airspeed = buf.airspeed.indicated_airspeed_m_s, + s += "f" #.true_airspeed = buf.airspeed.true_airspeed_m_s + s += "iii" # to align to 280 + d = struct.unpack(s, data) + return d + +def _main(): + if len(sys.argv) < 2: + print "Usage:\npython logconv.py <log.bin>" + return + fn = sys.argv[1] + sysvector_size = 280 + f = open(fn, "r") + while True: + data = f.read(sysvector_size) + if len(data) < sysvector_size: + break + a = [] + for i in _unpack_packet(data): + a.append(str(i)) + print ";".join(a) + f.close() + +if __name__ == "__main__": + _main() diff --git a/Tools/sdlog2_dump.py b/Tools/sdlog2_dump.py new file mode 100644 index 000000000..318f72971 --- /dev/null +++ b/Tools/sdlog2_dump.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python + +"""Dump binary log generated by sdlog2 or APM as CSV + +Usage: python sdlog2_dump.py <log.bin> [-v] [-e] [-d delimiter] [-n null] [-m MSG[.field1,field2,...]] + + -v Use plain debug output instead of CSV. + + -e Recover from errors. + + -d Use "delimiter" in CSV. Default is ",". + + -n Use "null" as placeholder for empty values in CSV. Default is empty. + + -m MSG[.field1,field2,...] + Dump only messages of specified type, and only specified fields. + Multiple -m options allowed.""" + +__author__ = "Anton Babushkin" +__version__ = "1.2" + +import struct, sys + +class SDLog2Parser: + BLOCK_SIZE = 8192 + MSG_HEADER_LEN = 3 + MSG_HEAD1 = 0xA3 + MSG_HEAD2 = 0x95 + MSG_FORMAT_PACKET_LEN = 89 + MSG_FORMAT_STRUCT = "BB4s16s64s" + MSG_TYPE_FORMAT = 0x80 + FORMAT_TO_STRUCT = { + "b": ("b", None), + "B": ("B", None), + "h": ("h", None), + "H": ("H", None), + "i": ("i", None), + "I": ("I", None), + "f": ("f", None), + "n": ("4s", None), + "N": ("16s", None), + "Z": ("64s", None), + "c": ("h", 0.01), + "C": ("H", 0.01), + "e": ("i", 0.01), + "E": ("I", 0.01), + "L": ("i", 0.0000001), + "M": ("b", None), + "q": ("q", None), + "Q": ("Q", None), + } + __csv_delim = "," + __csv_null = "" + __msg_filter = [] + __time_msg = None + __debug_out = False + __correct_errors = False + + def __init__(self): + return + + def reset(self): + self.__msg_descrs = {} # message descriptions by message type map + self.__msg_labels = {} # message labels by message name map + self.__msg_names = [] # message names in the same order as FORMAT messages + self.__buffer = "" # buffer for input binary data + self.__ptr = 0 # read pointer in buffer + self.__csv_columns = [] # CSV file columns in correct order in format "MSG.label" + self.__csv_data = {} # current values for all columns + self.__csv_updated = False + self.__msg_filter_map = {} # filter in form of map, with '*" expanded to full list of fields + + def setCSVDelimiter(self, csv_delim): + self.__csv_delim = csv_delim + + def setCSVNull(self, csv_null): + self.__csv_null = csv_null + + def setMsgFilter(self, msg_filter): + self.__msg_filter = msg_filter + + def setTimeMsg(self, time_msg): + self.__time_msg = time_msg + + def setDebugOut(self, debug_out): + self.__debug_out = debug_out + + def setCorrectErrors(self, correct_errors): + self.__correct_errors = correct_errors + + def process(self, fn): + self.reset() + if self.__debug_out: + # init __msg_filter_map + for msg_name, show_fields in self.__msg_filter: + self.__msg_filter_map[msg_name] = show_fields + first_data_msg = True + f = open(fn, "r") + bytes_read = 0 + while True: + chunk = f.read(self.BLOCK_SIZE) + if len(chunk) == 0: + break + self.__buffer = self.__buffer[self.__ptr:] + chunk + self.__ptr = 0 + while self.__bytesLeft() >= self.MSG_HEADER_LEN: + head1 = ord(self.__buffer[self.__ptr]) + head2 = ord(self.__buffer[self.__ptr+1]) + if (head1 != self.MSG_HEAD1 or head2 != self.MSG_HEAD2): + if self.__correct_errors: + self.__ptr += 1 + continue + else: + raise Exception("Invalid header at %i (0x%X): %02X %02X, must be %02X %02X" % (bytes_read + self.__ptr, bytes_read + self.__ptr, head1, head2, self.MSG_HEAD1, self.MSG_HEAD2)) + msg_type = ord(self.__buffer[self.__ptr+2]) + if msg_type == self.MSG_TYPE_FORMAT: + # parse FORMAT message + if self.__bytesLeft() < self.MSG_FORMAT_PACKET_LEN: + break + self.__parseMsgDescr() + else: + # parse data message + msg_descr = self.__msg_descrs[msg_type] + if msg_descr == None: + raise Exception("Unknown msg type: %i" % msg_type) + msg_length = msg_descr[0] + if self.__bytesLeft() < msg_length: + break + if first_data_msg: + # build CSV columns and init data map + self.__initCSV() + first_data_msg = False + self.__parseMsg(msg_descr) + bytes_read += self.__ptr + if not self.__debug_out and self.__time_msg != None and self.__csv_updated: + self.__printCSVRow() + f.close() + + def __bytesLeft(self): + return len(self.__buffer) - self.__ptr + + def __filterMsg(self, msg_name): + show_fields = "*" + if len(self.__msg_filter_map) > 0: + show_fields = self.__msg_filter_map.get(msg_name) + return show_fields + + def __initCSV(self): + if len(self.__msg_filter) == 0: + for msg_name in self.__msg_names: + self.__msg_filter.append((msg_name, "*")) + for msg_name, show_fields in self.__msg_filter: + if show_fields == "*": + show_fields = self.__msg_labels.get(msg_name, []) + self.__msg_filter_map[msg_name] = show_fields + for field in show_fields: + full_label = msg_name + "." + field + self.__csv_columns.append(full_label) + self.__csv_data[full_label] = None + print self.__csv_delim.join(self.__csv_columns) + + def __printCSVRow(self): + s = [] + for full_label in self.__csv_columns: + v = self.__csv_data[full_label] + if v == None: + v = self.__csv_null + else: + v = str(v) + s.append(v) + print self.__csv_delim.join(s) + + def __parseMsgDescr(self): + data = struct.unpack(self.MSG_FORMAT_STRUCT, self.__buffer[self.__ptr + 3 : self.__ptr + self.MSG_FORMAT_PACKET_LEN]) + msg_type = data[0] + if msg_type != self.MSG_TYPE_FORMAT: + msg_length = data[1] + msg_name = data[2].strip("\0") + msg_format = data[3].strip("\0") + msg_labels = data[4].strip("\0").split(",") + # Convert msg_format to struct.unpack format string + msg_struct = "" + msg_mults = [] + for c in msg_format: + try: + f = self.FORMAT_TO_STRUCT[c] + msg_struct += f[0] + msg_mults.append(f[1]) + except KeyError as e: + raise Exception("Unsupported format char: %s in message %s (%i)" % (c, msg_name, msg_type)) + msg_struct = "<" + msg_struct # force little-endian + self.__msg_descrs[msg_type] = (msg_length, msg_name, msg_format, msg_labels, msg_struct, msg_mults) + self.__msg_labels[msg_name] = msg_labels + self.__msg_names.append(msg_name) + if self.__debug_out: + if self.__filterMsg(msg_name) != None: + print "MSG FORMAT: type = %i, length = %i, name = %s, format = %s, labels = %s, struct = %s, mults = %s" % ( + msg_type, msg_length, msg_name, msg_format, str(msg_labels), msg_struct, msg_mults) + self.__ptr += self.MSG_FORMAT_PACKET_LEN + + def __parseMsg(self, msg_descr): + msg_length, msg_name, msg_format, msg_labels, msg_struct, msg_mults = msg_descr + if not self.__debug_out and self.__time_msg != None and msg_name == self.__time_msg and self.__csv_updated: + self.__printCSVRow() + self.__csv_updated = False + show_fields = self.__filterMsg(msg_name) + if (show_fields != None): + data = list(struct.unpack(msg_struct, self.__buffer[self.__ptr+self.MSG_HEADER_LEN:self.__ptr+msg_length])) + for i in xrange(len(data)): + if type(data[i]) is str: + data[i] = data[i].strip("\0") + m = msg_mults[i] + if m != None: + data[i] = data[i] * m + if self.__debug_out: + s = [] + for i in xrange(len(data)): + label = msg_labels[i] + if show_fields == "*" or label in show_fields: + s.append(label + "=" + str(data[i])) + print "MSG %s: %s" % (msg_name, ", ".join(s)) + else: + # update CSV data buffer + for i in xrange(len(data)): + label = msg_labels[i] + if label in show_fields: + self.__csv_data[msg_name + "." + label] = data[i] + if self.__time_msg != None and msg_name != self.__time_msg: + self.__csv_updated = True + if self.__time_msg == None: + self.__printCSVRow() + self.__ptr += msg_length + +def _main(): + if len(sys.argv) < 2: + print "Usage: python sdlog2_dump.py <log.bin> [-v] [-e] [-d delimiter] [-n null] [-m MSG[.field1,field2,...]] [-t TIME_MSG_NAME]\n" + print "\t-v\tUse plain debug output instead of CSV.\n" + print "\t-e\tRecover from errors.\n" + print "\t-d\tUse \"delimiter\" in CSV. Default is \",\".\n" + print "\t-n\tUse \"null\" as placeholder for empty values in CSV. Default is empty.\n" + print "\t-m MSG[.field1,field2,...]\n\t\tDump only messages of specified type, and only specified fields.\n\t\tMultiple -m options allowed." + print "\t-t\tSpecify TIME message name to group data messages by time and significantly reduce duplicate output.\n" + return + fn = sys.argv[1] + debug_out = False + correct_errors = False + msg_filter = [] + csv_null = "" + csv_delim = "," + time_msg = None + opt = None + for arg in sys.argv[2:]: + if opt != None: + if opt == "d": + csv_delim = arg + elif opt == "n": + csv_null = arg + elif opt == "t": + time_msg = arg + elif opt == "m": + show_fields = "*" + a = arg.split(".") + if len(a) > 1: + show_fields = a[1].split(",") + msg_filter.append((a[0], show_fields)) + opt = None + else: + if arg == "-v": + debug_out = True + elif arg == "-e": + correct_errors = True + elif arg == "-d": + opt = "d" + elif arg == "-n": + opt = "n" + elif arg == "-m": + opt = "m" + elif arg == "-t": + opt = "t" + + if csv_delim == "\\t": + csv_delim = "\t" + parser = SDLog2Parser() + parser.setCSVDelimiter(csv_delim) + parser.setCSVNull(csv_null) + parser.setMsgFilter(msg_filter) + parser.setTimeMsg(time_msg) + parser.setDebugOut(debug_out) + parser.setCorrectErrors(correct_errors) + parser.process(fn) + +if __name__ == "__main__": + _main() |