diff options
author | Dan O'Reilly <oreilldf@gmail.com> | 2015-08-12 23:57:46 -0400 |
---|---|---|
committer | Dan O'Reilly <oreilldf@gmail.com> | 2015-08-12 23:57:46 -0400 |
commit | e47cdd5a559f488ba52756927ce68f4cf93874fa (patch) | |
tree | 8ce2723e822808baf58e96f569c86035717ea351 /objectivec/DevTools | |
parent | daeaa6a28b81195f24d89222e649d79c9555af8b (diff) | |
parent | 38a56ee4b19d72c2e9d81a08b018704d1addf561 (diff) | |
download | protobuf-e47cdd5a559f488ba52756927ce68f4cf93874fa.tar.gz protobuf-e47cdd5a559f488ba52756927ce68f4cf93874fa.tar.bz2 protobuf-e47cdd5a559f488ba52756927ce68f4cf93874fa.zip |
Merge remote-tracking branch 'upstream/master' into py2_py3_straddle
Conflicts:
python/google/protobuf/descriptor_pool.py
python/google/protobuf/internal/api_implementation_default_test.py
python/google/protobuf/internal/cpp_message.py
python/google/protobuf/internal/descriptor_database_test.py
python/google/protobuf/internal/descriptor_pool_test.py
python/google/protobuf/internal/descriptor_python_test.py
python/google/protobuf/internal/descriptor_test.py
python/google/protobuf/internal/generator_test.py
python/google/protobuf/internal/message_factory_python_test.py
python/google/protobuf/internal/message_factory_test.py
python/google/protobuf/internal/message_test.py
python/google/protobuf/internal/proto_builder_test.py
python/google/protobuf/internal/python_message.py
python/google/protobuf/internal/reflection_test.py
python/google/protobuf/internal/service_reflection_test.py
python/google/protobuf/internal/symbol_database_test.py
python/google/protobuf/internal/text_encoding_test.py
python/google/protobuf/internal/text_format_test.py
python/google/protobuf/internal/unknown_fields_test.py
python/google/protobuf/internal/wire_format_test.py
python/google/protobuf/pyext/descriptor_cpp2_test.py
python/google/protobuf/pyext/message_factory_cpp2_test.py
python/google/protobuf/pyext/reflection_cpp2_generated_test.py
python/setup.py
ruby/lib/google/protobuf/message_exts.rb
Diffstat (limited to 'objectivec/DevTools')
-rwxr-xr-x | objectivec/DevTools/check_version_stamps.sh | 56 | ||||
-rwxr-xr-x | objectivec/DevTools/compile_testing_protos.sh | 113 | ||||
-rwxr-xr-x | objectivec/DevTools/full_mac_build.sh | 228 | ||||
-rwxr-xr-x | objectivec/DevTools/pddm.py | 687 | ||||
-rwxr-xr-x | objectivec/DevTools/pddm_tests.py | 515 |
5 files changed, 1599 insertions, 0 deletions
diff --git a/objectivec/DevTools/check_version_stamps.sh b/objectivec/DevTools/check_version_stamps.sh new file mode 100755 index 00000000..325b71dd --- /dev/null +++ b/objectivec/DevTools/check_version_stamps.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# This script checks that the runtime version number constant in the compiler +# source and in the runtime source is the same. +# +# A distro can be made of the protobuf sources with only a subset of the +# languages, so if the compiler depended on the Objective C runtime, those +# builds would break. At the same time, we don't want the runtime source +# depending on the compiler sources; so two copies of the constant are needed. + +set -eu + +readonly ScriptDir=$(dirname "$(echo $0 | sed -e "s,^\([^/]\),$(pwd)/\1,")") +readonly ProtoRootDir="${ScriptDir}/../.." + +die() { + echo "Error: $1" + exit 1 +} + +readonly ConstantName=GOOGLE_PROTOBUF_OBJC_GEN_VERSION + +# Collect version from plugin sources. + +readonly PluginSrc="${ProtoRootDir}/src/google/protobuf/compiler/objectivec/objectivec_file.cc" +readonly PluginVersion=$( \ + cat "${PluginSrc}" \ + | sed -n -e "s:const int32 ${ConstantName} = \([0-9]*\);:\1:p" +) + +if [[ -z "${PluginVersion}" ]] ; then + die "Failed to find ${ConstantName} in the plugin source (${PluginSrc})." +fi + +# Collect version from runtime sources. + +readonly RuntimeSrc="${ProtoRootDir}/objectivec/GPBBootstrap.h" +readonly RuntimeVersion=$( \ + cat "${RuntimeSrc}" \ + | sed -n -e "s:#define ${ConstantName} \([0-9]*\):\1:p" +) + +if [[ -z "${RuntimeVersion}" ]] ; then + die "Failed to find ${ConstantName} in the runtime source (${RuntimeSrc})." +fi + +# Compare them. + +if [[ "${PluginVersion}" != "${RuntimeVersion}" ]] ; then + die "Versions don't match! + Plugin: ${PluginVersion} from ${PluginSrc} + Runtime: ${RuntimeVersion} from ${RuntimeSrc} +" +fi + +# Success diff --git a/objectivec/DevTools/compile_testing_protos.sh b/objectivec/DevTools/compile_testing_protos.sh new file mode 100755 index 00000000..f414bd87 --- /dev/null +++ b/objectivec/DevTools/compile_testing_protos.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +# Invoked by the Xcode projects to build the protos needed for the unittests. + +set -eu + +readonly OUTPUT_DIR="${PROJECT_DERIVED_FILE_DIR}/protos" + +# Helper for bailing. +die() { + echo "Error: $1" + exit 2 +} + +# What to do. +case "${ACTION}" in + "") + # Build, fall thru + ;; + "clean") + rm -rf "${OUTPUT_DIR}" + exit 0 + ;; + *) + die "Unknown action requested: ${ACTION}" + ;; +esac + +# Move to the top of the protobuf directories. +cd "${SRCROOT}/.." + +[[ -x src/protoc ]] || \ + die "Could not find the protoc binary; make sure you have built it (objectivec/DevTools/full_mac_build.sh -h)." + +RUN_PROTOC=no +if [[ ! -d "${OUTPUT_DIR}" ]] ; then + RUN_PROTOC=yes +else + # Find the newest input file (protos, compiler, and this script). + # (these patterns catch some extra stuff, but better to over sample than + # under) + readonly NewestInput=$(find \ + src/google/protobuf/*.proto \ + objectivec/Tests/*.proto \ + src/.libs src/*.la src/protoc \ + objectivec/DevTools/compile_testing_protos.sh \ + -type f -print0 \ + | xargs -0 stat -f "%m %N" \ + | sort -n | tail -n1 | cut -f2- -d" ") + # Find the oldest output file. + readonly OldestOutput=$(find \ + "${OUTPUT_DIR}" \ + -type f -print0 \ + | xargs -0 stat -f "%m %N" \ + | sort -n -r | tail -n1 | cut -f2- -d" ") + # If the newest input is newer than the oldest output, regenerate. + if [[ "${NewestInput}" -nt "${OldestOutput}" ]] ; then + RUN_PROTOC=yes + fi +fi + +if [[ "${RUN_PROTOC}" != "yes" ]] ; then + # Up to date. + exit 0 +fi + +# Ensure the output dir exists +mkdir -p "${OUTPUT_DIR}/google/protobuf" + +CORE_PROTO_FILES=( \ + src/google/protobuf/unittest_custom_options.proto \ + src/google/protobuf/unittest_enormous_descriptor.proto \ + src/google/protobuf/unittest_embed_optimize_for.proto \ + src/google/protobuf/unittest_empty.proto \ + src/google/protobuf/unittest_import.proto \ + src/google/protobuf/unittest_import_lite.proto \ + src/google/protobuf/unittest_lite.proto \ + src/google/protobuf/unittest_mset.proto \ + src/google/protobuf/unittest_no_generic_services.proto \ + src/google/protobuf/unittest_optimize_for.proto \ + src/google/protobuf/unittest.proto \ + src/google/protobuf/unittest_import_public.proto \ + src/google/protobuf/unittest_import_public_lite.proto \ + src/google/protobuf/unittest_drop_unknown_fields.proto \ + src/google/protobuf/unittest_preserve_unknown_enum.proto \ + src/google/protobuf/map_lite_unittest.proto \ + src/google/protobuf/map_proto2_unittest.proto \ + src/google/protobuf/map_unittest.proto \ +) + +compile_proto() { + src/protoc \ + --objc_out="${OUTPUT_DIR}/google/protobuf" \ + --proto_path=src/google/protobuf/ \ + --proto_path=src \ + $* +} + +for a_proto in "${CORE_PROTO_FILES[@]}" ; do + compile_proto "${a_proto}" +done + +OBJC_PROTO_FILES=( \ + objectivec/Tests/unittest_cycle.proto \ + objectivec/Tests/unittest_runtime_proto2.proto \ + objectivec/Tests/unittest_runtime_proto3.proto \ + objectivec/Tests/unittest_objc.proto \ + objectivec/Tests/unittest_objc_startup.proto \ +) + +for a_proto in "${OBJC_PROTO_FILES[@]}" ; do + compile_proto --proto_path="objectivec/Tests" "${a_proto}" +done diff --git a/objectivec/DevTools/full_mac_build.sh b/objectivec/DevTools/full_mac_build.sh new file mode 100755 index 00000000..2192b760 --- /dev/null +++ b/objectivec/DevTools/full_mac_build.sh @@ -0,0 +1,228 @@ +#!/bin/bash +# +# Helper to do build so you don't have to remember all the steps/args. + + +set -eu + +# Some base locations. +readonly ScriptDir=$(dirname "$(echo $0 | sed -e "s,^\([^/]\),$(pwd)/\1,")") +readonly ProtoRootDir="${ScriptDir}/../.." + +printUsage() { + NAME=$(basename "${0}") + cat << EOF +usage: ${NAME} [OPTIONS] + +This script does the common build steps needed. + +OPTIONS: + + General: + + -h, --help + Show this message + -c, --clean + Issue a clean before the normal build. + -a, --autogen + Start by rerunning autogen & configure. + -r, --regenerate-descriptors + The descriptor.proto is checked in generated, cause it to regenerate. + -j #, --jobs # + Force the number of parallel jobs (useful for debugging build issues). + --skip-xcode + Skip the invoke of Xcode to test the runtime on both iOS and OS X. + --skip-xcode-ios + Skip the invoke of Xcode to test the runtime on iOS. + --skip-xcode-osx + Skip the invoke of Xcode to test the runtime on OS X. + +EOF +} + +header() { + echo "" + echo "========================================================================" + echo " ${@}" + echo "========================================================================" +} + +# Thanks to libtool, builds can fail in odd ways and since it eats some output +# it can be hard to spot, so force error output if make exits with a non zero. +wrapped_make() { + set +e # Don't stop if the command fails. + make $* + MAKE_EXIT_STATUS=$? + if [ ${MAKE_EXIT_STATUS} -ne 0 ]; then + echo "Error: 'make $*' exited with status ${MAKE_EXIT_STATUS}" + exit ${MAKE_EXIT_STATUS} + fi + set -e +} + +NUM_MAKE_JOBS=$(/usr/sbin/sysctl -n hw.ncpu) +if [[ "${NUM_MAKE_JOBS}" -lt 4 ]] ; then + NUM_MAKE_JOBS=4 +fi + +DO_AUTOGEN=no +DO_CLEAN=no +REGEN_CPP_DESCRIPTORS=no +DO_XCODE_IOS_TESTS=yes +DO_XCODE_OSX_TESTS=yes +while [[ $# != 0 ]]; do + case "${1}" in + -h | --help ) + printUsage + exit 0 + ;; + -c | --clean ) + DO_CLEAN=yes + ;; + -a | --autogen ) + DO_AUTOGEN=yes + ;; + -r | --regenerate-cpp-descriptors ) + REGEN_CPP_DESCRIPTORS=yes + ;; + -j | --jobs ) + shift + NUM_MAKE_JOBS="${1}" + ;; + --skip-xcode ) + DO_XCODE_IOS_TESTS=no + DO_XCODE_OSX_TESTS=no + ;; + --skip-xcode-ios ) + DO_XCODE_IOS_TESTS=no + ;; + --skip-xcode-osx ) + DO_XCODE_OSX_TESTS=no + ;; + -*) + echo "ERROR: Unknown option: ${1}" 1>&2 + printUsage + exit 1 + ;; + *) + echo "ERROR: Unknown argument: ${1}" 1>&2 + printUsage + exit 1 + ;; + esac + shift +done + +# Into the proto dir. +pushd "${ProtoRootDir}" + +# if no Makefile, force the autogen. +if [[ ! -f Makefile ]] ; then + DO_AUTOGEN=yes +fi + +if [[ "${DO_AUTOGEN}" == "yes" ]] ; then + header "Running autogen & configure" + ./autogen.sh + ./configure CXXFLAGS="-mmacosx-version-min=10.9 -Wnon-virtual-dtor -Woverloaded-virtual -Wunused-const-variable -Wunused-function" +fi + +if [[ "${DO_CLEAN}" == "yes" ]] ; then + header "Cleaning" + wrapped_make clean + if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then + XCODEBUILD_CLEAN_BASE_IOS=( + xcodebuild + -project objectivec/ProtocolBuffers_iOS.xcodeproj + -scheme ProtocolBuffers + ) + "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Debug clean + "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Release clean + fi + if [[ "${DO_XCODE_OSX_TESTS}" == "yes" ]] ; then + XCODEBUILD_CLEAN_BASE_OSX=( + xcodebuild + -project objectivec/ProtocolBuffers_OSX.xcodeproj + -scheme ProtocolBuffers + ) + "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Debug clean + "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Release clean + fi +fi + +if [[ "${REGEN_CPP_DESCRIPTORS}" == "yes" ]] ; then + header "Regenerating the C++ descriptor sources." + ./generate_descriptor_proto.sh -j "${NUM_MAKE_JOBS}" +fi + +header "Building" +# Can't issue these together, when fully parallel, something sometimes chokes +# at random. +wrapped_make -j "${NUM_MAKE_JOBS}" all +wrapped_make -j "${NUM_MAKE_JOBS}" check + +header "Ensuring the ObjC descriptors are current." +# Find the newest input file (protos, compiler, and the generator script). +# (these patterns catch some extra stuff, but better to over sample than under) +readonly NewestInput=$(find \ + src/google/protobuf/*.proto \ + src/.libs src/*.la src/protoc \ + objectivec/generate_descriptors_proto.sh \ + -type f -print0 \ + | xargs -0 stat -f "%m %N" \ + | sort -n | tail -n1 | cut -f2- -d" ") +# Find the oldest output file. +readonly OldestOutput=$(find \ + "${ProtoRootDir}/objectivec/google" \ + -type f -print0 \ + | xargs -0 stat -f "%m %N" \ + | sort -n -r | tail -n1 | cut -f2- -d" ") +# If the newest input is newer than the oldest output, regenerate. +if [[ "${NewestInput}" -nt "${OldestOutput}" ]] ; then + echo ">> Newest input is newer than oldest output, regenerating." + objectivec/generate_descriptors_proto.sh -j "${NUM_MAKE_JOBS}" +else + echo ">> Newest input is older than oldest output, no need to regenerating." +fi + +header "Checking on the ObjC Runtime Code" +objectivec/DevTools/pddm_tests.py +if ! objectivec/DevTools/pddm.py --dry-run objectivec/*.[hm] objectivec/Tests/*.[hm] ; then + echo "" + echo "Update by running:" + echo " objectivec/DevTools/pddm.py objectivec/*.[hm] objectivec/Tests/*.[hm]" + exit 1 +fi + +if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then + XCODEBUILD_TEST_BASE_IOS=( + xcodebuild + -project objectivec/ProtocolBuffers_iOS.xcodeproj + -scheme ProtocolBuffers + # Don't need to worry about form factors or retina/non retina; + # just pick a mix of OS Versions and 32/64 bit. + -destination "platform=iOS Simulator,name=iPhone 4s,OS=7.1" # 32bit + -destination "platform=iOS Simulator,name=iPhone 6,OS=8.3" # 64bit + -destination "platform=iOS Simulator,name=iPad 2,OS=7.1" # 32bit + -destination "platform=iOS Simulator,name=iPad Air,OS=8.3" # 64bit + ) + header "Doing Xcode iOS build/tests - Debug" + "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Debug test + header "Doing Xcode iOS build/tests - Release" + "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Release test + # Don't leave the simulator in the developer's face. + killall "iOS Simulator" +fi +if [[ "${DO_XCODE_OSX_TESTS}" == "yes" ]] ; then + XCODEBUILD_TEST_BASE_OSX=( + xcodebuild + -project objectivec/ProtocolBuffers_OSX.xcodeproj + -scheme ProtocolBuffers + # Since the ObjC 2.0 Runtime is required, 32bit OS X isn't supported. + -destination "platform=OS X,arch=x86_64" # 64bit + ) + header "Doing Xcode OS X build/tests - Debug" + "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Debug test + header "Doing Xcode OS X build/tests - Release" + "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Release test +fi diff --git a/objectivec/DevTools/pddm.py b/objectivec/DevTools/pddm.py new file mode 100755 index 00000000..d1b53f5a --- /dev/null +++ b/objectivec/DevTools/pddm.py @@ -0,0 +1,687 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2015 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""PDDM - Poor Developers' Debug-able Macros + +A simple markup that can be added in comments of source so they can then be +expanded out into code. Most of this could be done with CPP macros, but then +developers can't really step through them in the debugger, this way they are +expanded to the same code, but you can debug them. + +Any file can be processed, but the syntax is designed around a C based compiler. +Processed lines start with "//%". There are three types of sections you can +create: Text (left alone), Macro Definitions, and Macro Expansions. There is +no order required between definitions and expansions, all definitions are read +before any expansions are processed (thus, if desired, definitions can be put +at the end of the file to keep them out of the way of the code). + +Macro Definitions are started with "//%PDDM-DEFINE Name(args)" and all lines +afterwards that start with "//%" are included in the definition. Multiple +macros can be defined in one block by just using a another "//%PDDM-DEFINE" +line to start the next macro. Optionally, a macro can be ended with +"//%PDDM-DEFINE-END", this can be useful when you want to make it clear that +trailing blank lines are included in the macro. You can also end a definition +with an expansion. + +Macro Expansions are started by single lines containing +"//%PDDM-EXPAND Name(args)" and then with "//%PDDM-EXPAND-END" or another +expansions. All lines in-between are replaced by the result of the expansion. +The first line of the expansion is always a blank like just for readability. + +Expansion itself is pretty simple, one macro can invoke another macro, but +you cannot nest the invoke of a macro in another macro (i.e. - can't do +"foo(bar(a))", but you can define foo(a) and bar(b) where bar invokes foo() +within its expansion. + +When macros are expanded, the arg references can also add "$O" suffix to the +name (i.e. - "NAME$O") to specify an option to be applied. The options are: + + $S - Replace each character in the value with a space. + $l - Lowercase the first letter of the value. + $L - Lowercase the whole value. + $u - Uppercase the first letter of the value. + $U - Uppercase the whole value. + +Within a macro you can use ## to cause things to get joined together after +expansion (i.e. - "a##b" within a macro will become "ab"). + +Example: + + int foo(MyEnum x) { + switch (x) { + //%PDDM-EXPAND case(Enum_Left, 1) + //%PDDM-EXPAND case(Enum_Center, 2) + //%PDDM-EXPAND case(Enum_Right, 3) + //%PDDM-EXPAND-END + } + + //%PDDM-DEFINE case(_A, _B) + //% case _A: + //% return _B; + + A macro ends at the start of the next one, or an optional %PDDM-DEFINE-END + can be used to avoid adding extra blank lines/returns (or make it clear when + it is desired). + + One macro can invoke another by simply using its name NAME(ARGS). You cannot + nest an invoke inside another (i.e. - NAME1(NAME2(ARGS)) isn't supported). + + Within a macro you can use ## to cause things to get joined together after + processing (i.e. - "a##b" within a macro will become "ab"). + + +""" + +import optparse +import os +import re +import sys + + +# Regex for macro definition. +_MACRO_RE = re.compile(r'(?P<name>\w+)\((?P<args>.*?)\)') +# Regex for macro's argument definition. +_MACRO_ARG_NAME_RE = re.compile(r'^\w+$') + +# Line inserted after each EXPAND. +_GENERATED_CODE_LINE = ( + '// This block of code is generated, do not edit it directly.' +) + + +def _MacroRefRe(macro_names): + # Takes in a list of macro names and makes a regex that will match invokes + # of those macros. + return re.compile(r'\b(?P<macro_ref>(?P<name>(%s))\((?P<args>.*?)\))' % + '|'.join(macro_names)) + +def _MacroArgRefRe(macro_arg_names): + # Takes in a list of macro arg names and makes a regex that will match + # uses of those args. + return re.compile(r'\b(?P<name>(%s))(\$(?P<option>.))?\b' % + '|'.join(macro_arg_names)) + + +class PDDMError(Exception): + """Error thrown by pddm.""" + pass + + +class MacroCollection(object): + """Hold a set of macros and can resolve/expand them.""" + + def __init__(self, a_file=None): + """Initializes the collection. + + Args: + a_file: The file like stream to parse. + + Raises: + PDDMError if there are any issues. + """ + self._macros = dict() + if a_file: + self.ParseInput(a_file) + + class MacroDefinition(object): + """Holds a macro definition.""" + + def __init__(self, name, arg_names): + self._name = name + self._args = tuple(arg_names) + self._body = '' + self._needNewLine = False + + def AppendLine(self, line): + if self._needNewLine: + self._body += '\n' + self._body += line + self._needNewLine = not line.endswith('\n') + + @property + def name(self): + return self._name + + @property + def args(self): + return self._args + + @property + def body(self): + return self._body + + def ParseInput(self, a_file): + """Consumes input extracting definitions. + + Args: + a_file: The file like stream to parse. + + Raises: + PDDMError if there are any issues. + """ + input_lines = a_file.read().splitlines() + self.ParseLines(input_lines) + + def ParseLines(self, input_lines): + """Parses list of lines. + + Args: + input_lines: A list of strings of input to parse (no newlines on the + strings). + + Raises: + PDDMError if there are any issues. + """ + current_macro = None + for line in input_lines: + if line.startswith('PDDM-'): + directive = line.split(' ', 1)[0] + if directive == 'PDDM-DEFINE': + name, args = self._ParseDefineLine(line) + if self._macros.get(name): + raise PDDMError('Attempt to redefine macro: "%s"' % line) + current_macro = self.MacroDefinition(name, args) + self._macros[name] = current_macro + continue + if directive == 'PDDM-DEFINE-END': + if not current_macro: + raise PDDMError('Got DEFINE-END directive without an active macro:' + ' "%s"' % line) + current_macro = None + continue + raise PDDMError('Hit a line with an unknown directive: "%s"' % line) + + if current_macro: + current_macro.AppendLine(line) + continue + + # Allow blank lines between macro definitions. + if line.strip() == '': + continue + + raise PDDMError('Hit a line that wasn\'t a directive and no open macro' + ' definition: "%s"' % line) + + def _ParseDefineLine(self, input_line): + assert input_line.startswith('PDDM-DEFINE') + line = input_line[12:].strip() + match = _MACRO_RE.match(line) + # Must match full line + if match is None or match.group(0) != line: + raise PDDMError('Failed to parse macro definition: "%s"' % input_line) + name = match.group('name') + args_str = match.group('args').strip() + args = [] + if args_str: + for part in args_str.split(','): + arg = part.strip() + if arg == '': + raise PDDMError('Empty arg name in macro definition: "%s"' + % input_line) + if not _MACRO_ARG_NAME_RE.match(arg): + raise PDDMError('Invalid arg name "%s" in macro definition: "%s"' + % (arg, input_line)) + if arg in args: + raise PDDMError('Arg name "%s" used more than once in macro' + ' definition: "%s"' % (arg, input_line)) + args.append(arg) + return (name, tuple(args)) + + def Expand(self, macro_ref_str): + """Expands the macro reference. + + Args: + macro_ref_str: String of a macro reference (i.e. foo(a, b)). + + Returns: + The text from the expansion. + + Raises: + PDDMError if there are any issues. + """ + match = _MACRO_RE.match(macro_ref_str) + if match is None or match.group(0) != macro_ref_str: + raise PDDMError('Failed to parse macro reference: "%s"' % macro_ref_str) + if match.group('name') not in self._macros: + raise PDDMError('No macro named "%s".' % match.group('name')) + return self._Expand(match, [], macro_ref_str) + + def _FormatStack(self, macro_ref_stack): + result = '' + for _, macro_ref in reversed(macro_ref_stack): + result += '\n...while expanding "%s".' % macro_ref + return result + + def _Expand(self, macro_ref_match, macro_stack, macro_ref_str=None): + if macro_ref_str is None: + macro_ref_str = macro_ref_match.group('macro_ref') + name = macro_ref_match.group('name') + for prev_name, prev_macro_ref in macro_stack: + if name == prev_name: + raise PDDMError('Found macro recusion, invoking "%s":%s' % + (macro_ref_str, self._FormatStack(macro_stack))) + macro = self._macros[name] + args_str = macro_ref_match.group('args').strip() + args = [] + if args_str or len(macro.args): + args = [x.strip() for x in args_str.split(',')] + if len(args) != len(macro.args): + raise PDDMError('Expected %d args, got: "%s".%s' % + (len(macro.args), macro_ref_str, + self._FormatStack(macro_stack))) + # Replace args usages. + result = self._ReplaceArgValues(macro, args, macro_ref_str, macro_stack) + # Expand any macro invokes. + new_macro_stack = macro_stack + [(name, macro_ref_str)] + while True: + eval_result = self._EvalMacrosRefs(result, new_macro_stack) + # Consume all ## directives to glue things together. + eval_result = eval_result.replace('##', '') + if eval_result == result: + break + result = eval_result + return result + + def _ReplaceArgValues(self, + macro, arg_values, macro_ref_to_report, macro_stack): + if len(arg_values) == 0: + # Nothing to do + return macro.body + assert len(arg_values) == len(macro.args) + args = dict(zip(macro.args, arg_values)) + def _lookupArg(match): + val = args[match.group('name')] + opt = match.group('option') + if opt: + if opt == 'S': # Spaces for the length + return ' ' * len(val) + elif opt == 'l': # Lowercase first character + if val: + return val[0].lower() + val[1:] + else: + return val + elif opt == 'L': # All Lowercase + return val.lower() + elif opt == 'u': # Uppercase first character + if val: + return val[0].upper() + val[1:] + else: + return val + elif opt == 'U': # All Uppercase + return val.upper() + else: + raise PDDMError('Unknown arg option "%s$%s" while expanding "%s".%s' + % (match.group('name'), match.group('option'), + macro_ref_to_report, + self._FormatStack(macro_stack))) + return val + # Let the regex do the work! + macro_arg_ref_re = _MacroArgRefRe(macro.args) + return macro_arg_ref_re.sub(_lookupArg, macro.body) + + def _EvalMacrosRefs(self, text, macro_stack): + macro_ref_re = _MacroRefRe(self._macros.keys()) + def _resolveMacro(match): + return self._Expand(match, macro_stack) + return macro_ref_re.sub(_resolveMacro, text) + + +class SourceFile(object): + """Represents a source file with PDDM directives in it.""" + + def __init__(self, a_file, import_resolver=None): + """Initializes the file reading in the file. + + Args: + a_file: The file to read in. + import_resolver: a function that given a path will return a stream for + the contents. + + Raises: + PDDMError if there are any issues. + """ + self._sections = [] + self._original_content = a_file.read() + self._import_resolver = import_resolver + self._processed_content = None + + class SectionBase(object): + + def __init__(self, first_line_num): + self._lines = [] + self._first_line_num = first_line_num + + def TryAppend(self, line, line_num): + """Try appending a line. + + Args: + line: The line to append. + line_num: The number of the line. + + Returns: + A tuple of (SUCCESS, CAN_ADD_MORE). If SUCCESS if False, the line + wasn't append. If SUCCESS is True, then CAN_ADD_MORE is True/False to + indicate if more lines can be added after this one. + """ + assert False, "sublcass should have overridden" + return (False, False) + + def HitEOF(self): + """Called when the EOF was reached for for a given section.""" + pass + + def BindMacroCollection(self, macro_collection): + """Binds the chunk to a macro collection. + + Args: + macro_collection: The collection to bind too. + """ + pass + + def Append(self, line): + self._lines.append(line) + + @property + def lines(self): + return self._lines + + @property + def num_lines_captured(self): + return len(self._lines) + + @property + def first_line_num(self): + return self._first_line_num + + @property + def first_line(self): + if not self._lines: + return '' + return self._lines[0] + + @property + def text(self): + return '\n'.join(self.lines) + '\n' + + class TextSection(SectionBase): + """Text section that is echoed out as is.""" + + def TryAppend(self, line, line_num): + if line.startswith('//%PDDM'): + return (False, False) + self.Append(line) + return (True, True) + + class ExpansionSection(SectionBase): + """Section that is the result of an macro expansion.""" + + def __init__(self, first_line_num): + SourceFile.SectionBase.__init__(self, first_line_num) + self._macro_collection = None + + def TryAppend(self, line, line_num): + if line.startswith('//%PDDM'): + directive = line.split(' ', 1)[0] + if directive == '//%PDDM-EXPAND': + self.Append(line) + return (True, True) + if directive == '//%PDDM-EXPAND-END': + assert self.num_lines_captured > 0 + return (True, False) + raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' % + (directive, line_num, self.first_line)) + # Eat other lines. + return (True, True) + + def HitEOF(self): + raise PDDMError('Hit the end of the file while in "%s".' % + self.first_line) + + def BindMacroCollection(self, macro_collection): + self._macro_collection = macro_collection + + @property + def lines(self): + captured_lines = SourceFile.SectionBase.lines.fget(self) + directive_len = len('//%PDDM-EXPAND') + result = [] + for line in captured_lines: + result.append(line) + if self._macro_collection: + # Always add a blank line, seems to read better. (If need be, add an + # option to the EXPAND to indicate if this should be done.) + result.extend([_GENERATED_CODE_LINE, '']) + macro = line[directive_len:].strip() + try: + expand_result = self._macro_collection.Expand(macro) + # Since expansions are line oriented, strip trailing whitespace + # from the lines. + lines = [x.rstrip() for x in expand_result.split('\n')] + result.append('\n'.join(lines)) + except PDDMError as e: + raise PDDMError('%s\n...while expanding "%s" from the section' + ' that started:\n Line %d: %s' % + (e.message, macro, + self.first_line_num, self.first_line)) + + # Add the ending marker. + if len(captured_lines) == 1: + result.append('//%%PDDM-EXPAND-END %s' % + captured_lines[0][directive_len:].strip()) + else: + result.append('//%%PDDM-EXPAND-END (%s expansions)' % len(captured_lines)) + + return result + + class DefinitionSection(SectionBase): + """Section containing macro definitions""" + + def TryAppend(self, line, line_num): + if not line.startswith('//%'): + return (False, False) + if line.startswith('//%PDDM'): + directive = line.split(' ', 1)[0] + if directive == "//%PDDM-EXPAND": + return False, False + if directive not in ('//%PDDM-DEFINE', '//%PDDM-DEFINE-END'): + raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' % + (directive, line_num, self.first_line)) + self.Append(line) + return (True, True) + + def BindMacroCollection(self, macro_collection): + if macro_collection: + try: + # Parse the lines after stripping the prefix. + macro_collection.ParseLines([x[3:] for x in self.lines]) + except PDDMError as e: + raise PDDMError('%s\n...while parsing section that started:\n' + ' Line %d: %s' % + (e.message, self.first_line_num, self.first_line)) + + class ImportDefinesSection(SectionBase): + """Section containing an import of PDDM-DEFINES from an external file.""" + + def __init__(self, first_line_num, import_resolver): + SourceFile.SectionBase.__init__(self, first_line_num) + self._import_resolver = import_resolver + + def TryAppend(self, line, line_num): + if not line.startswith('//%PDDM-IMPORT-DEFINES '): + return (False, False) + assert self.num_lines_captured == 0 + self.Append(line) + return (True, False) + + def BindMacroCollection(self, macro_colletion): + if not macro_colletion: + return + if self._import_resolver is None: + raise PDDMError('Got an IMPORT-DEFINES without a resolver (line %d):' + ' "%s".' % (self.first_line_num, self.first_line)) + import_name = self.first_line.split(' ', 1)[1].strip() + imported_file = self._import_resolver(import_name) + if imported_file is None: + raise PDDMError('Resolver failed to find "%s" (line %d):' + ' "%s".' % + (import_name, self.first_line_num, self.first_line)) + try: + imported_src_file = SourceFile(imported_file, self._import_resolver) + imported_src_file._ParseFile() + for section in imported_src_file._sections: + section.BindMacroCollection(macro_colletion) + except PDDMError as e: + raise PDDMError('%s\n...while importing defines:\n' + ' Line %d: %s' % + (e.message, self.first_line_num, self.first_line)) + + def _ParseFile(self): + self._sections = [] + lines = self._original_content.splitlines() + cur_section = None + for line_num, line in enumerate(lines, 1): + if not cur_section: + cur_section = self._MakeSection(line, line_num) + was_added, accept_more = cur_section.TryAppend(line, line_num) + if not was_added: + cur_section = self._MakeSection(line, line_num) + was_added, accept_more = cur_section.TryAppend(line, line_num) + assert was_added + if not accept_more: + cur_section = None + + if cur_section: + cur_section.HitEOF() + + def _MakeSection(self, line, line_num): + if not line.startswith('//%PDDM'): + section = self.TextSection(line_num) + else: + directive = line.split(' ', 1)[0] + if directive == '//%PDDM-EXPAND': + section = self.ExpansionSection(line_num) + elif directive == '//%PDDM-DEFINE': + section = self.DefinitionSection(line_num) + elif directive == '//%PDDM-IMPORT-DEFINES': + section = self.ImportDefinesSection(line_num, self._import_resolver) + else: + raise PDDMError('Unexpected line %d: "%s".' % (line_num, line)) + self._sections.append(section) + return section + + def ProcessContent(self, strip_expansion=False): + """Processes the file contents.""" + self._ParseFile() + if strip_expansion: + # Without a collection the expansions become blank, removing them. + collection = None + else: + collection = MacroCollection() + for section in self._sections: + section.BindMacroCollection(collection) + result = '' + for section in self._sections: + result += section.text + self._processed_content = result + + @property + def original_content(self): + return self._original_content + + @property + def processed_content(self): + return self._processed_content + + +def main(args): + usage = '%prog [OPTIONS] PATH ...' + description = ( + 'Processes PDDM directives in the the given paths and write them back' + ' out.' + ) + parser = optparse.OptionParser(usage=usage, description=description) + parser.add_option('--dry-run', + default=False, action='store_true', + help='Don\'t write back to the file(s), just report if the' + ' contents needs an update and exit with a value of 1.') + parser.add_option('--verbose', + default=False, action='store_true', + help='Reports is a file is already current.') + parser.add_option('--collapse', + default=False, action='store_true', + help='Removes all the generated code.') + opts, extra_args = parser.parse_args(args) + + if not extra_args: + parser.error('Need atleast one file to process') + + result = 0 + for a_path in extra_args: + if not os.path.exists(a_path): + sys.stderr.write('ERROR: File not found: %s\n' % a_path) + return 100 + + def _ImportResolver(name): + # resolve based on the file being read. + a_dir = os.path.dirname(a_path) + import_path = os.path.join(a_dir, name) + if not os.path.exists(import_path): + return None + return open(import_path, 'r') + + with open(a_path, 'r') as f: + src_file = SourceFile(f, _ImportResolver) + + try: + src_file.ProcessContent(strip_expansion=opts.collapse) + except PDDMError as e: + sys.stderr.write('ERROR: %s\n...While processing "%s"\n' % + (e.message, a_path)) + return 101 + + if src_file.processed_content != src_file.original_content: + if not opts.dry_run: + print 'Updating for "%s".' % a_path + with open(a_path, 'w') as f: + f.write(src_file.processed_content) + else: + # Special result to indicate things need updating. + print 'Update needed for "%s".' % a_path + result = 1 + elif opts.verbose: + print 'No update for "%s".' % a_path + + return result + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/objectivec/DevTools/pddm_tests.py b/objectivec/DevTools/pddm_tests.py new file mode 100755 index 00000000..8a73b842 --- /dev/null +++ b/objectivec/DevTools/pddm_tests.py @@ -0,0 +1,515 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2015 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Tests for pddm.py.""" + +import io +import unittest + +import pddm + + +class TestParsingMacros(unittest.TestCase): + + def testParseEmpty(self): + f = io.StringIO(u'') + result = pddm.MacroCollection(f) + self.assertEqual(len(result._macros), 0) + + def testParseOne(self): + f = io.StringIO(u"""PDDM-DEFINE foo( ) +body""") + result = pddm.MacroCollection(f) + self.assertEqual(len(result._macros), 1) + macro = result._macros.get('foo') + self.assertIsNotNone(macro) + self.assertEquals(macro.name, 'foo') + self.assertEquals(macro.args, tuple()) + self.assertEquals(macro.body, 'body') + + def testParseGeneral(self): + # Tests multiple defines, spaces in all places, etc. + f = io.StringIO(u""" +PDDM-DEFINE noArgs( ) +body1 +body2 + +PDDM-DEFINE-END + +PDDM-DEFINE oneArg(foo) +body3 +PDDM-DEFINE twoArgs( bar_ , baz ) +body4 +body5""") + result = pddm.MacroCollection(f) + self.assertEqual(len(result._macros), 3) + macro = result._macros.get('noArgs') + self.assertIsNotNone(macro) + self.assertEquals(macro.name, 'noArgs') + self.assertEquals(macro.args, tuple()) + self.assertEquals(macro.body, 'body1\nbody2\n') + macro = result._macros.get('oneArg') + self.assertIsNotNone(macro) + self.assertEquals(macro.name, 'oneArg') + self.assertEquals(macro.args, ('foo',)) + self.assertEquals(macro.body, 'body3') + macro = result._macros.get('twoArgs') + self.assertIsNotNone(macro) + self.assertEquals(macro.name, 'twoArgs') + self.assertEquals(macro.args, ('bar_', 'baz')) + self.assertEquals(macro.body, 'body4\nbody5') + # Add into existing collection + f = io.StringIO(u""" +PDDM-DEFINE another(a,b,c) +body1 +body2""") + result.ParseInput(f) + self.assertEqual(len(result._macros), 4) + macro = result._macros.get('another') + self.assertIsNotNone(macro) + self.assertEquals(macro.name, 'another') + self.assertEquals(macro.args, ('a', 'b', 'c')) + self.assertEquals(macro.body, 'body1\nbody2') + + def testParseDirectiveIssues(self): + test_list = [ + # Unknown directive + (u'PDDM-DEFINE foo()\nbody\nPDDM-DEFINED foo\nbaz', + 'Hit a line with an unknown directive: '), + # End without begin + (u'PDDM-DEFINE foo()\nbody\nPDDM-DEFINE-END\nPDDM-DEFINE-END\n', + 'Got DEFINE-END directive without an active macro: '), + # Line not in macro block + (u'PDDM-DEFINE foo()\nbody\nPDDM-DEFINE-END\nmumble\n', + 'Hit a line that wasn\'t a directive and no open macro definition: '), + # Redefine macro + (u'PDDM-DEFINE foo()\nbody\nPDDM-DEFINE foo(a)\nmumble\n', + 'Attempt to redefine macro: '), + ] + for idx, (input_str, expected_prefix) in enumerate(test_list, 1): + f = io.StringIO(input_str) + try: + result = pddm.MacroCollection(f) + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertTrue(e.message.startswith(expected_prefix), + 'Entry %d failed: %r' % (idx, e)) + + def testParseBeginIssues(self): + test_list = [ + # 1. No name + (u'PDDM-DEFINE\nmumble', + 'Failed to parse macro definition: '), + # 2. No name (with spaces) + (u'PDDM-DEFINE \nmumble', + 'Failed to parse macro definition: '), + # 3. No open paren + (u'PDDM-DEFINE foo\nmumble', + 'Failed to parse macro definition: '), + # 4. No close paren + (u'PDDM-DEFINE foo(\nmumble', + 'Failed to parse macro definition: '), + # 5. No close paren (with args) + (u'PDDM-DEFINE foo(a, b\nmumble', + 'Failed to parse macro definition: '), + # 6. No name before args + (u'PDDM-DEFINE (a, b)\nmumble', + 'Failed to parse macro definition: '), + # 7. No name before args + (u'PDDM-DEFINE foo bar(a, b)\nmumble', + 'Failed to parse macro definition: '), + # 8. Empty arg name + (u'PDDM-DEFINE foo(a, ,b)\nmumble', + 'Empty arg name in macro definition: '), + (u'PDDM-DEFINE foo(a,,b)\nmumble', + 'Empty arg name in macro definition: '), + # 10. Duplicate name + (u'PDDM-DEFINE foo(a,b,a,c)\nmumble', + 'Arg name "a" used more than once in macro definition: '), + # 11. Invalid arg name + (u'PDDM-DEFINE foo(a b,c)\nmumble', + 'Invalid arg name "a b" in macro definition: '), + (u'PDDM-DEFINE foo(a.b,c)\nmumble', + 'Invalid arg name "a.b" in macro definition: '), + (u'PDDM-DEFINE foo(a-b,c)\nmumble', + 'Invalid arg name "a-b" in macro definition: '), + (u'PDDM-DEFINE foo(a,b,c.)\nmumble', + 'Invalid arg name "c." in macro definition: '), + # 15. Extra stuff after the name + (u'PDDM-DEFINE foo(a,c) foo\nmumble', + 'Failed to parse macro definition: '), + (u'PDDM-DEFINE foo(a,c) foo)\nmumble', + 'Failed to parse macro definition: '), + ] + for idx, (input_str, expected_prefix) in enumerate(test_list, 1): + f = io.StringIO(input_str) + try: + result = pddm.MacroCollection(f) + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertTrue(e.message.startswith(expected_prefix), + 'Entry %d failed: %r' % (idx, e)) + + +class TestExpandingMacros(unittest.TestCase): + + def testExpandBasics(self): + f = io.StringIO(u""" +PDDM-DEFINE noArgs( ) +body1 +body2 + +PDDM-DEFINE-END + +PDDM-DEFINE oneArg(a) +body3 a + +PDDM-DEFINE-END + +PDDM-DEFINE twoArgs(b,c) +body4 b c +body5 +PDDM-DEFINE-END + +""") + mc = pddm.MacroCollection(f) + test_list = [ + (u'noArgs()', + 'body1\nbody2\n'), + (u'oneArg(wee)', + 'body3 wee\n'), + (u'twoArgs(having some, fun)', + 'body4 having some fun\nbody5'), + # One arg, pass empty. + (u'oneArg()', + 'body3 \n'), + # Two args, gets empty in each slot. + (u'twoArgs(, empty)', + 'body4 empty\nbody5'), + (u'twoArgs(empty, )', + 'body4 empty \nbody5'), + (u'twoArgs(, )', + 'body4 \nbody5'), + ] + for idx, (input_str, expected) in enumerate(test_list, 1): + result = mc.Expand(input_str) + self.assertEqual(result, expected, + 'Entry %d --\n Result: %r\n Expected: %r' % + (idx, result, expected)) + + def testExpandArgOptions(self): + f = io.StringIO(u""" +PDDM-DEFINE bar(a) +a-a$S-a$l-a$L-a$u-a$U +PDDM-DEFINE-END +""") + mc = pddm.MacroCollection(f) + + self.assertEqual(mc.Expand('bar(xYz)'), 'xYz- -xYz-xyz-XYz-XYZ') + self.assertEqual(mc.Expand('bar(MnoP)'), 'MnoP- -mnoP-mnop-MnoP-MNOP') + # Test empty + self.assertEqual(mc.Expand('bar()'), '-----') + + def testExpandSimpleMacroErrors(self): + f = io.StringIO(u""" +PDDM-DEFINE foo(a, b) +<a-z> +PDDM-DEFINE baz(a) +a - a$z +""") + mc = pddm.MacroCollection(f) + test_list = [ + # 1. Unknown macro + (u'bar()', + 'No macro named "bar".'), + (u'bar(a)', + 'No macro named "bar".'), + # 3. Arg mismatch + (u'foo()', + 'Expected 2 args, got: "foo()".'), + (u'foo(a b)', + 'Expected 2 args, got: "foo(a b)".'), + (u'foo(a,b,c)', + 'Expected 2 args, got: "foo(a,b,c)".'), + # 6. Unknown option in expansion + (u'baz(mumble)', + 'Unknown arg option "a$z" while expanding "baz(mumble)".'), + ] + for idx, (input_str, expected_err) in enumerate(test_list, 1): + try: + result = mc.Expand(input_str) + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertEqual(e.message, expected_err, + 'Entry %d failed: %r' % (idx, e)) + + def testExpandReferences(self): + f = io.StringIO(u""" +PDDM-DEFINE StartIt() +foo(abc, def) +foo(ghi, jkl) +PDDM-DEFINE foo(a, b) +bar(a, int) +bar(b, NSString *) +PDDM-DEFINE bar(n, t) +- (t)n; +- (void)set##n$u##:(t)value; + +""") + mc = pddm.MacroCollection(f) + expected = """- (int)abc; +- (void)setAbc:(int)value; + +- (NSString *)def; +- (void)setDef:(NSString *)value; + +- (int)ghi; +- (void)setGhi:(int)value; + +- (NSString *)jkl; +- (void)setJkl:(NSString *)value; +""" + self.assertEqual(mc.Expand('StartIt()'), expected) + + def testCatchRecursion(self): + f = io.StringIO(u""" +PDDM-DEFINE foo(a, b) +bar(1, a) +bar(2, b) +PDDM-DEFINE bar(x, y) +foo(x, y) +""") + mc = pddm.MacroCollection(f) + try: + result = mc.Expand('foo(A,B)') + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertEqual(e.message, + 'Found macro recusion, invoking "foo(1, A)":\n...while expanding "bar(1, A)".\n...while expanding "foo(A,B)".') + + +class TestParsingSource(unittest.TestCase): + + def testBasicParse(self): + test_list = [ + # 1. no directives + (u'a\nb\nc', + (3,) ), + # 2. One define + (u'a\n//%PDDM-DEFINE foo()\n//%body\nc', + (1, 2, 1) ), + # 3. Two defines + (u'a\n//%PDDM-DEFINE foo()\n//%body\n//%PDDM-DEFINE bar()\n//%body2\nc', + (1, 4, 1) ), + # 4. Two defines with ends + (u'a\n//%PDDM-DEFINE foo()\n//%body\n//%PDDM-DEFINE-END\n' + u'//%PDDM-DEFINE bar()\n//%body2\n//%PDDM-DEFINE-END\nc', + (1, 6, 1) ), + # 5. One expand, one define (that runs to end of file) + (u'a\n//%PDDM-EXPAND foo()\nbody\n//%PDDM-EXPAND-END\n' + u'//%PDDM-DEFINE bar()\n//%body2\n', + (1, 1, 2) ), + # 6. One define ended with an expand. + (u'a\nb\n//%PDDM-DEFINE bar()\n//%body2\n' + u'//%PDDM-EXPAND bar()\nbody2\n//%PDDM-EXPAND-END\n', + (2, 2, 1) ), + # 7. Two expands (one end), one define. + (u'a\n//%PDDM-EXPAND foo(1)\nbody\n//%PDDM-EXPAND foo(2)\nbody2\n//%PDDM-EXPAND-END\n' + u'//%PDDM-DEFINE foo()\n//%body2\n', + (1, 2, 2) ), + ] + for idx, (input_str, line_counts) in enumerate(test_list, 1): + f = io.StringIO(input_str) + sf = pddm.SourceFile(f) + sf._ParseFile() + self.assertEqual(len(sf._sections), len(line_counts), + 'Entry %d -- %d != %d' % + (idx, len(sf._sections), len(line_counts))) + for idx2, (sec, expected) in enumerate(zip(sf._sections, line_counts), 1): + self.assertEqual(sec.num_lines_captured, expected, + 'Entry %d, section %d -- %d != %d' % + (idx, idx2, sec.num_lines_captured, expected)) + + def testErrors(self): + test_list = [ + # 1. Directive within expansion + (u'//%PDDM-EXPAND a()\n//%PDDM-BOGUS', + 'Ran into directive ("//%PDDM-BOGUS", line 2) while in "//%PDDM-EXPAND a()".'), + (u'//%PDDM-EXPAND a()\n//%PDDM-DEFINE a()\n//%body\n', + 'Ran into directive ("//%PDDM-DEFINE", line 2) while in "//%PDDM-EXPAND a()".'), + # 3. Expansion ran off end of file + (u'//%PDDM-EXPAND a()\na\nb\n', + 'Hit the end of the file while in "//%PDDM-EXPAND a()".'), + # 4. Directive within define + (u'//%PDDM-DEFINE a()\n//%body\n//%PDDM-BOGUS', + 'Ran into directive ("//%PDDM-BOGUS", line 3) while in "//%PDDM-DEFINE a()".'), + (u'//%PDDM-DEFINE a()\n//%body\n//%PDDM-EXPAND-END a()', + 'Ran into directive ("//%PDDM-EXPAND-END", line 3) while in "//%PDDM-DEFINE a()".'), + # 6. Directives that shouldn't start sections + (u'a\n//%PDDM-DEFINE-END a()\n//a\n', + 'Unexpected line 2: "//%PDDM-DEFINE-END a()".'), + (u'a\n//%PDDM-EXPAND-END a()\n//a\n', + 'Unexpected line 2: "//%PDDM-EXPAND-END a()".'), + (u'//%PDDM-BOGUS\n//a\n', + 'Unexpected line 1: "//%PDDM-BOGUS".'), + ] + for idx, (input_str, expected_err) in enumerate(test_list, 1): + f = io.StringIO(input_str) + try: + pddm.SourceFile(f)._ParseFile() + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertEqual(e.message, expected_err, + 'Entry %d failed: %r' % (idx, e)) + +class TestProcessingSource(unittest.TestCase): + + def testBasics(self): + input_str = u""" +//%PDDM-IMPORT-DEFINES ImportFile +foo +//%PDDM-EXPAND mumble(abc) +//%PDDM-EXPAND-END +bar +//%PDDM-EXPAND mumble(def) +//%PDDM-EXPAND mumble(ghi) +//%PDDM-EXPAND-END +baz +//%PDDM-DEFINE mumble(a_) +//%a_: getName(a_) +""" + input_str2 = u""" +//%PDDM-DEFINE getName(x_) +//%do##x_$u##(int x_); + +""" + expected = u""" +//%PDDM-IMPORT-DEFINES ImportFile +foo +//%PDDM-EXPAND mumble(abc) +// This block of code is generated, do not edit it directly. + +abc: doAbc(int abc); +//%PDDM-EXPAND-END mumble(abc) +bar +//%PDDM-EXPAND mumble(def) +// This block of code is generated, do not edit it directly. + +def: doDef(int def); +//%PDDM-EXPAND mumble(ghi) +// This block of code is generated, do not edit it directly. + +ghi: doGhi(int ghi); +//%PDDM-EXPAND-END (2 expansions) +baz +//%PDDM-DEFINE mumble(a_) +//%a_: getName(a_) +""" + expected_stripped = u""" +//%PDDM-IMPORT-DEFINES ImportFile +foo +//%PDDM-EXPAND mumble(abc) +//%PDDM-EXPAND-END mumble(abc) +bar +//%PDDM-EXPAND mumble(def) +//%PDDM-EXPAND mumble(ghi) +//%PDDM-EXPAND-END (2 expansions) +baz +//%PDDM-DEFINE mumble(a_) +//%a_: getName(a_) +""" + def _Resolver(name): + self.assertEqual(name, 'ImportFile') + return io.StringIO(input_str2) + f = io.StringIO(input_str) + sf = pddm.SourceFile(f, _Resolver) + sf.ProcessContent() + self.assertEqual(sf.processed_content, expected) + # Feed it through and nothing should change. + f2 = io.StringIO(sf.processed_content) + sf2 = pddm.SourceFile(f2, _Resolver) + sf2.ProcessContent() + self.assertEqual(sf2.processed_content, expected) + self.assertEqual(sf2.processed_content, sf.processed_content) + # Test stripping (with the original input and expanded version). + f2 = io.StringIO(input_str) + sf2 = pddm.SourceFile(f2) + sf2.ProcessContent(strip_expansion=True) + self.assertEqual(sf2.processed_content, expected_stripped) + f2 = io.StringIO(sf.processed_content) + sf2 = pddm.SourceFile(f2, _Resolver) + sf2.ProcessContent(strip_expansion=True) + self.assertEqual(sf2.processed_content, expected_stripped) + + def testProcessFileWithMacroParseError(self): + input_str = u""" +foo +//%PDDM-DEFINE mumble(a_) +//%body +//%PDDM-DEFINE mumble(x_) +//%body2 + +""" + f = io.StringIO(input_str) + sf = pddm.SourceFile(f) + try: + sf.ProcessContent() + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertEqual(e.message, + 'Attempt to redefine macro: "PDDM-DEFINE mumble(x_)"\n' + '...while parsing section that started:\n' + ' Line 3: //%PDDM-DEFINE mumble(a_)') + + def testProcessFileWithExpandError(self): + input_str = u""" +foo +//%PDDM-DEFINE mumble(a_) +//%body +//%PDDM-EXPAND foobar(x_) +//%PDDM-EXPAND-END + +""" + f = io.StringIO(input_str) + sf = pddm.SourceFile(f) + try: + sf.ProcessContent() + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertEqual(e.message, + 'No macro named "foobar".\n' + '...while expanding "foobar(x_)" from the section that' + ' started:\n Line 5: //%PDDM-EXPAND foobar(x_)') + + +if __name__ == '__main__': + unittest.main() |