Merge remote-tracking branch 'upstream/master' into py2_py3_straddle

Conflicts:
	python/google/protobuf/descriptor_pool.py
	python/google/protobuf/internal/api_implementation_default_test.py
	python/google/protobuf/internal/cpp_message.py
	python/google/protobuf/internal/descriptor_database_test.py
	python/google/protobuf/internal/descriptor_pool_test.py
	python/google/protobuf/internal/descriptor_python_test.py
	python/google/protobuf/internal/descriptor_test.py
	python/google/protobuf/internal/generator_test.py
	python/google/protobuf/internal/message_factory_python_test.py
	python/google/protobuf/internal/message_factory_test.py
	python/google/protobuf/internal/message_test.py
	python/google/protobuf/internal/proto_builder_test.py
	python/google/protobuf/internal/python_message.py
	python/google/protobuf/internal/reflection_test.py
	python/google/protobuf/internal/service_reflection_test.py
	python/google/protobuf/internal/symbol_database_test.py
	python/google/protobuf/internal/text_encoding_test.py
	python/google/protobuf/internal/text_format_test.py
	python/google/protobuf/internal/unknown_fields_test.py
	python/google/protobuf/internal/wire_format_test.py
	python/google/protobuf/pyext/descriptor_cpp2_test.py
	python/google/protobuf/pyext/message_factory_cpp2_test.py
	python/google/protobuf/pyext/reflection_cpp2_generated_test.py
	python/setup.py
	ruby/lib/google/protobuf/message_exts.rb

1 files changed, 182 insertions, 0 deletions

diff --git a/update_file_lists.sh b/update_file_lists.sh
new file mode 100755
index 00000000..34443456
--- /dev/null
+++ b/update_file_lists.sh
@@ -0,0 +1,182 @@
+#!/bin/sh
+
+# This script copies source file lists from src/Makefile.am to cmake files.
+
+get_variable_value() {
+  local FILENAME=$1
+  local VARNAME=$2
+  awk "
+    BEGIN { start = 0; }
+    /^$VARNAME =/ { start = 1; }
+    { if (start) { print \$0; } }
+    /\\\\\$/ { next; }
+    { start = 0; }
+  " $FILENAME \
+    | sed "s/^$VARNAME =//" \
+    | sed "s/[ \\]//g" \
+    | grep -v "^\\$" \
+    | grep -v "^$" \
+    | LC_ALL=C sort | uniq
+}
+
+get_header_files() {
+  get_variable_value $@ | grep '\.h$'
+}
+
+get_source_files() {
+  get_variable_value $@ | grep "cc$"
+}
+
+get_proto_files() {
+  get_variable_value $@ | grep "pb.cc$" | sed "s/pb.cc/proto/"
+}
+
+sort_files() {
+  for FILE in $@; do
+    echo $FILE
+  done | LC_ALL=C sort | uniq
+}
+
+MAKEFILE=src/Makefile.am
+
+[ -f "$MAKEFILE" ] || {
+  echo "Cannot find: $MAKEFILE"
+  exit 1
+}
+
+# Extract file lists from src/Makefile.am
+GZHEADERS=$(get_variable_value $MAKEFILE GZHEADERS)
+HEADERS=$(get_variable_value $MAKEFILE nobase_include_HEADERS)
+PUBLIC_HEADERS=$(sort_files $GZHEADERS $HEADERS)
+LIBPROTOBUF_LITE_SOURCES=$(get_source_files $MAKEFILE libprotobuf_lite_la_SOURCES)
+LIBPROTOBUF_SOURCES=$(get_source_files $MAKEFILE libprotobuf_la_SOURCES)
+LIBPROTOC_SOURCES=$(get_source_files $MAKEFILE libprotoc_la_SOURCES)
+LITE_PROTOS=$(get_proto_files $MAKEFILE protoc_lite_outputs)
+PROTOS=$(get_proto_files $MAKEFILE protoc_outputs)
+WKT_PROTOS=$(get_variable_value $MAKEFILE nobase_dist_proto_DATA)
+COMMON_TEST_SOURCES=$(get_source_files $MAKEFILE COMMON_TEST_SOURCES)
+TEST_SOURCES=$(get_source_files $MAKEFILE protobuf_test_SOURCES)
+LITE_TEST_SOURCES=$(get_source_files $MAKEFILE protobuf_lite_test_SOURCES)
+TEST_PLUGIN_SOURCES=$(get_source_files $MAKEFILE test_plugin_SOURCES)
+
+################################################################################
+# Update cmake files.
+################################################################################
+
+CMAKE_DIR=cmake
+EXTRACT_INCLUDES_BAT=cmake/extract_includes.bat.in
+[ -d "$CMAKE_DIR" ] || {
+  echo "Cannot find: $CMAKE_DIR"
+  exit 1
+}
+
+[ -f "$EXTRACT_INCLUDES_BAT" ] || {
+  echo "Cannot find: $EXTRACT_INCLUDES_BAT"
+  exit 1
+}
+
+set_cmake_value() {
+  local FILENAME=$1
+  local VARNAME=$2
+  local PREFIX=$3
+  shift
+  shift
+  shift
+  awk -v values="$*" -v prefix="$PREFIX" "
+    BEGIN { start = 0; }
+    /^set\\($VARNAME/ {
+      start = 1;
+      print \$0;
+      len = split(values, vlist, \" \");
+      for (i = 1; i <= len; ++i) {
+        printf(\"  %s%s\\n\", prefix, vlist[i]);
+      }
+      next;
+    }
+    start && /^\\)/ {
+      start = 0;
+    }
+    !start {
+      print \$0;
+    }
+  " $FILENAME > /tmp/$$
+  cp /tmp/$$ $FILENAME
+}
+
+
+# Replace file lists in cmake files.
+CMAKE_PREFIX="\${protobuf_source_dir}/src/"
+set_cmake_value $CMAKE_DIR/libprotobuf-lite.cmake libprotobuf_lite_files $CMAKE_PREFIX $LIBPROTOBUF_LITE_SOURCES
+set_cmake_value $CMAKE_DIR/libprotobuf.cmake libprotobuf_files $CMAKE_PREFIX $LIBPROTOBUF_SOURCES
+set_cmake_value $CMAKE_DIR/libprotoc.cmake libprotoc_files $CMAKE_PREFIX $LIBPROTOC_SOURCES
+set_cmake_value $CMAKE_DIR/tests.cmake lite_test_protos "" $LITE_PROTOS
+set_cmake_value $CMAKE_DIR/tests.cmake tests_protos "" $PROTOS
+set_cmake_value $CMAKE_DIR/tests.cmake common_test_files $CMAKE_PREFIX $COMMON_TEST_SOURCES
+set_cmake_value $CMAKE_DIR/tests.cmake tests_files $CMAKE_PREFIX $TEST_SOURCES
+set_cmake_value $CMAKE_DIR/tests.cmake lite_test_files $CMAKE_PREFIX $LITE_TEST_SOURCES
+
+# Generate extract_includes.bat
+echo "mkdir include" > $EXTRACT_INCLUDES_BAT
+for HEADER in $PUBLIC_HEADERS; do
+  HEADER_DIR=$(dirname $HEADER)
+  while [ ! "$HEADER_DIR" = "." ]; do
+    echo $HEADER_DIR | sed "s/\\//\\\\/g"
+    HEADER_DIR=$(dirname $HEADER_DIR)
+  done
+done | sort | uniq | sed "s/^/mkdir include\\\\/" >> $EXTRACT_INCLUDES_BAT
+for HEADER in $PUBLIC_HEADERS; do
+  WINPATH=$(echo $HEADER | sed 's;/;\\;g')
+  echo "copy \${PROTOBUF_SOURCE_WIN32_PATH}\\..\\src\\$WINPATH include\\$WINPATH" >> $EXTRACT_INCLUDES_BAT
+done
+
+################################################################################
+# Update bazel BUILD files.
+################################################################################
+
+BAZEL_BUILD=./BUILD
+[ -f "$BAZEL_BUILD" ] || {
+  echo "Cannot find: $BAZEL_BUILD"
+  exit 1
+}
+set_bazel_value() {
+  local FILENAME=$1
+  local VARNAME=$2
+  local PREFIX=$3
+  shift
+  shift
+  shift
+  awk -v values="$*" -v prefix="$PREFIX" "
+    BEGIN { start = 0; }
+    /# AUTOGEN\\($VARNAME\\)/ {
+      start = 1;
+      print \$0;
+      # replace \$0 with indent.
+      sub(/#.*/, \"\", \$0)
+      len = split(values, vlist, \" \");
+      for (i = 1; i <= len; ++i) {
+        printf(\"%s\\\"%s%s\\\",\n\", \$0, prefix, vlist[i]);
+      }
+      next;
+    }
+    start && /\]/ {
+      start = 0
+    }
+    !start {
+      print \$0;
+    }
+  " $FILENAME > /tmp/$$
+  cp /tmp/$$ $FILENAME
+}
+
+
+BAZEL_PREFIX="src/"
+set_bazel_value $BAZEL_BUILD protobuf_lite_srcs $BAZEL_PREFIX $LIBPROTOBUF_LITE_SOURCES
+set_bazel_value $BAZEL_BUILD protobuf_srcs $BAZEL_PREFIX $LIBPROTOBUF_SOURCES
+set_bazel_value $BAZEL_BUILD protoc_lib_srcs $BAZEL_PREFIX $LIBPROTOC_SOURCES
+set_bazel_value $BAZEL_BUILD lite_test_protos "" $LITE_PROTOS
+set_bazel_value $BAZEL_BUILD well_known_protos "" $WKT_PROTOS
+set_bazel_value $BAZEL_BUILD test_protos "" $PROTOS
+set_bazel_value $BAZEL_BUILD common_test_srcs $BAZEL_PREFIX $COMMON_TEST_SOURCES
+set_bazel_value $BAZEL_BUILD test_srcs $BAZEL_PREFIX $TEST_SOURCES
+set_bazel_value $BAZEL_BUILD test_plugin_srcs $BAZEL_PREFIX $TEST_PLUGIN_SOURCES
+