Parameters list generator

7 files changed, 378 insertions, 0 deletions

diff --git a/Tools/px4params/README.md b/Tools/px4params/README.md
new file mode 100644
index 000000000..a23b44799
--- /dev/null
+++ b/Tools/px4params/README.md
@@ -0,0 +1,9 @@
+h1. PX4 Parameters Processor
+
+It's designed to scan PX4 source codes, find declarations of tunable parameters,
+and generate the list in various formats.
+
+Currently supported formats are:
+
+* XML for the parametric UI generator
+* Human-readable description in DokuWiki format
diff --git a/Tools/px4params/dokuwikiout.py b/Tools/px4params/dokuwikiout.py
new file mode 100644
index 000000000..33f76b415
--- /dev/null
+++ b/Tools/px4params/dokuwikiout.py
@@ -0,0 +1,27 @@
+import output
+
+class DokuWikiOutput(output.Output):
+    def Generate(self, groups):
+        result = ""
+        for group in groups:
+            result += "==== %s ====\n\n" % group.GetName()
+            for param in group.GetParams():
+                code = param.GetFieldValue("code")
+                name = param.GetFieldValue("short_desc")
+                if code != name:
+                    name = "%s (%s)" % (name, code)
+                result += "=== %s ===\n\n" % name
+                long_desc = param.GetFieldValue("long_desc")
+                if long_desc is not None:
+                    result += "%s\n\n" % long_desc
+                min_val = param.GetFieldValue("min")
+                if min_val is not None:
+                    result += "* Minimal value: %s\n" % min_val
+                max_val = param.GetFieldValue("max")
+                if max_val is not None:
+                    result += "* Maximal value: %s\n" % max_val
+                def_val = param.GetFieldValue("default")
+                if def_val is not None:
+                    result += "* Default value: %s\n" % def_val
+                result += "\n"
+        return result
diff --git a/Tools/px4params/output.py b/Tools/px4params/output.py
new file mode 100644
index 000000000..c09246871
--- /dev/null
+++ b/Tools/px4params/output.py
@@ -0,0 +1,5 @@
+class Output(object):
+    def Save(self, groups, fn):
+        data = self.Generate(groups)
+        with open(fn, 'w') as f:
+            f.write(data)
diff --git a/Tools/px4params/parser.py b/Tools/px4params/parser.py
new file mode 100644
index 000000000..251be672f
--- /dev/null
+++ b/Tools/px4params/parser.py
@@ -0,0 +1,220 @@
+import sys
+import re
+
+class ParameterGroup(object):
+    """
+    Single parameter group
+    """
+    def __init__(self, name):
+        self.name = name
+        self.params = []
+
+    def AddParameter(self, param):
+        """
+        Add parameter to the group
+        """
+        self.params.append(param)
+
+    def GetName(self):
+        """
+        Get parameter group name
+        """
+        return self.name
+
+    def GetParams(self):
+        """
+        Returns the parsed list of parameters. Every parameter is a Parameter
+        object. Note that returned object is not a copy. Modifications affect
+        state of the parser.
+        """
+        return sorted(self.params,
+                cmp=lambda x, y: cmp(x.GetFieldValue("code"),
+                    y.GetFieldValue("code")))
+
+class Parameter(object):
+    """
+    Single parameter
+    """
+
+    # Define sorting order of the fields
+    priority = {
+        "code": 10,
+        "type": 9,
+        "short_desc": 8,
+        "long_desc": 7,
+        "default": 6,
+        "min": 5,
+        "max": 4,
+        # all others == 0 (sorted alphabetically)
+    }
+
+    def __init__(self):
+        self.fields = {}
+
+    def SetField(self, code, value):
+        """
+        Set named field value
+        """
+        self.fields[code] = value
+
+    def GetFieldCodes(self):
+        """
+        Return list of existing field codes in convenient order
+        """
+        return sorted(self.fields.keys(),
+                cmp=lambda x, y: cmp(self.priority.get(y, 0),
+                    self.priority.get(x, 0)) or cmp(x, y))
+
+    def GetFieldValue(self, code):
+        """
+        Return value of the given field code or None if not found.
+        """
+        return self.fields.get(code)
+
+class Parser(object):
+    """
+    Parses provided data and stores all found parameters internally.
+    """
+
+    re_split_lines = re.compile(r'[\r\n]+')
+    re_comment_start = re.compile(r'^\/\*\*')
+    re_comment_content = re.compile(r'^\*\s*(.*)') 
+    re_comment_tag = re.compile(r'@([a-zA-Z][a-zA-Z0-9_]*)\s*(.*)')
+    re_comment_end = re.compile(r'(.*?)\s*\*\/')
+    re_parameter_definition = re.compile(r'PARAM_DEFINE_([A-Z_][A-Z0-9_]*)\s*\(([A-Z_][A-Z0-9_]*)\s*,\s*([^ ,\)]+)\s*\)\s*;')
+    re_cut_type_specifier = re.compile(r'[a-z]+$')
+    re_is_a_number = re.compile(r'^-?[0-9\.]')
+    re_remove_dots = re.compile(r'\.+$')
+
+    valid_tags = set(["min", "max", "group"])
+
+    # Order of parameter groups
+    priority = {
+        # All other groups = 0 (sort alphabetically)
+        "Miscellaneous": -10
+    }
+
+    def __init__(self):
+        self.param_groups = {}
+
+    def GetSupportedExtensions(self):
+        """
+        Returns list of supported file extensions that can be parsed by this
+        parser.
+        """
+        return ["cpp", "c"]
+
+    def Parse(self, contents):
+        """
+        Incrementally parse program contents and append all found parameters
+        to the list.
+        """
+        # This code is essentially a comment-parsing grammar. "state"
+        # represents parser state. It contains human-readable state
+        # names.
+        state = None
+        for line in self.re_split_lines.split(contents):
+            line = line.strip()
+            # Ignore empty lines
+            if line == "":
+                continue
+            if self.re_comment_start.match(line):
+                state = "wait-short"
+                short_desc = None
+                long_desc = None
+                tags = {}
+            elif state is not None and state != "comment-processed":
+                m = self.re_comment_end.search(line)
+                if m:
+                    line = m.group(1)
+                    last_comment_line = True
+                else:
+                    last_comment_line = False
+                m = self.re_comment_content.match(line)
+                if m:
+                    comment_content = m.group(1)
+                    if comment_content == "":
+                        # When short comment ends with empty comment line,
+                        # start waiting for the next part - long comment.
+                        if state == "wait-short-end":
+                            state = "wait-long"
+                    else:
+                        m = self.re_comment_tag.match(comment_content)
+                        if m:
+                            tag, desc = m.group(1, 2)
+                            tags[tag] = desc
+                            current_tag = tag
+                            state = "wait-tag-end"
+                        elif state == "wait-short":
+                            # Store first line of the short description
+                            short_desc = comment_content
+                            state = "wait-short-end"
+                        elif state == "wait-short-end":
+                            # Append comment line to the short description
+                            short_desc += "\n" + comment_content
+                        elif state == "wait-long":
+                            # Store first line of the long description
+                            long_desc = comment_content
+                            state = "wait-long-end"
+                        elif state == "wait-long-end":
+                            # Append comment line to the long description
+                            long_desc += "\n" + comment_content
+                        elif state == "wait-tag-end":
+                            # Append comment line to the tag text
+                            tags[current_tag] += "\n" + comment_content
+                        else:
+                            raise AssertionError(
+                                    "Invalid parser state: %s" % state)
+                elif not last_comment_line:
+                    # Invalid comment line (inside comment, but not starting with
+                    # "*" or "*/". Reset parsed content.
+                    state = None
+                if last_comment_line:
+                    state = "comment-processed"
+            else:
+                # Non-empty line outside the comment
+                m = self.re_parameter_definition.match(line)
+                if m:
+                    tp, code, defval = m.group(1, 2, 3)
+                    # Remove trailing type specifier from numbers: 0.1f => 0.1
+                    if self.re_is_a_number.match(defval):
+                        defval = self.re_cut_type_specifier.sub('', defval)
+                    param = Parameter()
+                    param.SetField("code", code)
+                    param.SetField("short_desc", code)
+                    param.SetField("type", tp)
+                    param.SetField("default", defval)
+                    # If comment was found before the parameter declaration,
+                    # inject its data into the newly created parameter.
+                    group = "Miscellaneous"
+                    if state == "comment-processed":
+                        if short_desc is not None:
+                            param.SetField("short_desc",
+                                    self.re_remove_dots.sub('', short_desc))
+                        if long_desc is not None:
+                            param.SetField("long_desc", long_desc)
+                        for tag in tags:
+                            if tag == "group":
+                                group = tags[tag]
+                            elif tag not in self.valid_tags:
+                                sys.stderr.write("Skipping invalid"
+                                        "documentation tag: '%s'\n" % tag)
+                            else:
+                                param.SetField(tag, tags[tag])
+                    # Store the parameter
+                    if group not in self.param_groups:
+                        self.param_groups[group] = ParameterGroup(group)
+                    self.param_groups[group].AddParameter(param)
+                # Reset parsed comment.
+                state = None
+
+    def GetParamGroups(self):
+        """
+        Returns the parsed list of parameters. Every parameter is a Parameter
+        object. Note that returned object is not a copy. Modifications affect
+        state of the parser.
+        """
+        return sorted(self.param_groups.values(),
+                cmp=lambda x, y: cmp(self.priority.get(y.GetName(), 0),
+                    self.priority.get(x.GetName(), 0)) or cmp(x.GetName(),
+                        y.GetName()))
diff --git a/Tools/px4params/px_process_params.py b/Tools/px4params/px_process_params.py
new file mode 100755
index 000000000..cdf5aba7c
--- /dev/null
+++ b/Tools/px4params/px_process_params.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+############################################################################
+#
+#   Copyright (C) 2013 PX4 Development Team. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name PX4 nor the names of its contributors may be
+#    used to endorse or promote products derived from this software
+#    without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+############################################################################
+
+#
+# PX4 paramers processor (main executable file)
+#
+# It scans src/ subdirectory of the project, collects all parameters
+# definitions, and outputs list of parameters in XML and DokuWiki formats.
+#
+
+import scanner
+import parser
+import xmlout
+import dokuwikiout
+
+# Initialize parser
+prs = parser.Parser()
+
+# Scan directories, and parse the files
+sc = scanner.Scanner()
+sc.ScanDir("../../src", prs)
+output = prs.GetParamGroups()
+
+# Output into XML
+out = xmlout.XMLOutput()
+out.Save(output, "parameters.xml")
+
+# Output into DokuWiki
+out = dokuwikiout.DokuWikiOutput()
+out.Save(output, "parameters.wiki")
diff --git a/Tools/px4params/scanner.py b/Tools/px4params/scanner.py
new file mode 100644
index 000000000..b5a1af47c
--- /dev/null
+++ b/Tools/px4params/scanner.py
@@ -0,0 +1,34 @@
+import os
+import re
+
+class Scanner(object):
+    """
+    Traverses directory tree, reads all source files, and passes their contents
+    to the Parser.
+    """
+
+    re_file_extension = re.compile(r'\.([^\.]+)$')
+
+    def ScanDir(self, srcdir, parser):
+        """
+        Scans provided path and passes all found contents to the parser using
+        parser.Parse method.
+        """
+        extensions = set(parser.GetSupportedExtensions())
+        for dirname, dirnames, filenames in os.walk(srcdir):
+            for filename in filenames:
+                m = self.re_file_extension.search(filename)
+                if m:
+                    ext = m.group(1)
+                    if ext in extensions:
+                        path = os.path.join(dirname, filename)
+                        self.ScanFile(path, parser)
+
+    def ScanFile(self, path, parser):
+        """
+        Scans provided file and passes its contents to the parser using
+        parser.Parse method.
+        """
+        with open(path, 'r') as f:
+            contents = f.read()
+        parser.Parse(contents)
diff --git a/Tools/px4params/xmlout.py b/Tools/px4params/xmlout.py
new file mode 100644
index 000000000..d56802b15
--- /dev/null
+++ b/Tools/px4params/xmlout.py
@@ -0,0 +1,22 @@
+import output
+from xml.dom.minidom import getDOMImplementation
+
+class XMLOutput(output.Output):
+    def Generate(self, groups):
+        impl = getDOMImplementation()
+        xml_document = impl.createDocument(None, "parameters", None)
+        xml_parameters = xml_document.documentElement
+        for group in groups:
+            xml_group = xml_document.createElement("group")
+            xml_group.setAttribute("name", group.GetName())
+            xml_parameters.appendChild(xml_group)
+            for param in group.GetParams():
+                xml_param = xml_document.createElement("parameter")
+                xml_group.appendChild(xml_param)
+                for code in param.GetFieldCodes():
+                    value = param.GetFieldValue(code)
+                    xml_field = xml_document.createElement(code)
+                    xml_param.appendChild(xml_field)
+                    xml_value = xml_document.createTextNode(value)
+                    xml_field.appendChild(xml_value)
+        return xml_document.toprettyxml(indent="    ", newl="\n", encoding="utf-8")