From fa60e550ec8ca140f60a3cb8f36fa41fc45acb27 Mon Sep 17 00:00:00 2001 From: Yilun Chong Date: Tue, 9 Jan 2018 10:24:50 -0800 Subject: Fix java benchmark to use parser, fix cpp benchmark new arena to use Reset, format some files --- benchmarks/Makefile.am | 8 ++--- benchmarks/README.md | 41 +++++++++++----------- benchmarks/cpp_benchmark.cc | 3 +- benchmarks/java/pom.xml | 10 +++--- .../com/google/protobuf/ProtoBenchCaliper.java | 18 ++++------ benchmarks/py_benchmark.py | 40 +++++++++++---------- 6 files changed, 59 insertions(+), 61 deletions(-) diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index 08eae398..cfb1d5b9 100755 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -45,7 +45,7 @@ protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_pr protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) ) touch protoc_middleman2 - + endif all_data = `find . -type f -name "dataset.*.pb"` @@ -75,7 +75,7 @@ benchmarks_protoc_outputs_proto2_header = \ datasets/google_message4/benchmark_message4.pb.h \ datasets/google_message4/benchmark_message4_1.pb.h \ datasets/google_message4/benchmark_message4_2.pb.h \ - datasets/google_message4/benchmark_message4_3.pb.h + datasets/google_message4/benchmark_message4_3.pb.h benchmarks_protoc_outputs_proto2 = \ datasets/google_message1/benchmark_message1_proto2.pb.cc \ @@ -144,7 +144,7 @@ javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middlem cd ../.. @touch javac_middleman -java-benchmark: javac_middleman +java-benchmark: javac_middleman @echo "Writing shortcut script java-benchmark..." @echo '#! /bin/sh' > java-benchmark @echo 'java -cp '"tmp/java/target/*.jar"' com.google.protobuf.ProtoBench $$@' >> java-benchmark @@ -173,7 +173,7 @@ python_cpp_pkg_flags = `pkg-config --cflags --libs python` lib_LTLIBRARIES = libbenchmark_messages.la libbenchmark_messages_la_SOURCES = python_benchmark_messages.cc -libbenchmark_messages_la_LIBADD = $(top_srcdir)/src/.libs/libprotobuf.la +libbenchmark_messages_la_LIBADD = $(top_srcdir)/src/.libs/libprotobuf.la libbenchmark_messages_la_LDFLAGS = -version-info 1:0:0 -export-dynamic libbenchmark_messages_la_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) $(python_cpp_pkg_flags) libbenchmark_messages_la-libbenchmark_messages_la.$(OBJEXT): $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) diff --git a/benchmarks/README.md b/benchmarks/README.md index 71104d69..459c7b9d 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -7,45 +7,46 @@ protobuf language runtime. ## Prerequisite -First, you need to follow the instruction in the root directory's README to +First, you need to follow the instruction in the root directory's README to build your language's protobuf, then: ### CPP You need to install [cmake](https://cmake.org/) before building the benchmark. -We are using [google/benchmark](https://github.com/google/benchmark) as the -benchmark tool for testing cpp. This will be automaticly made during build the +We are using [google/benchmark](https://github.com/google/benchmark) as the +benchmark tool for testing cpp. This will be automaticly made during build the cpp benchmark. ### Java -We're using maven to build the java benchmarks, which is the same as to build -the Java protobuf. There're no other tools need to install. We're using -[google/caliper](https://github.com/google/caliper) as benchmark tool, which +We're using maven to build the java benchmarks, which is the same as to build +the Java protobuf. There're no other tools need to install. We're using +[google/caliper](https://github.com/google/caliper) as benchmark tool, which can be automaticly included by maven. ### Python -We're using python C++ API for testing the generated +We're using python C++ API for testing the generated CPP proto version of python protobuf, which is also a prerequisite for Python -protobuf cpp implementation. You need to install the correct version of Python -C++ extension package before run generated CPP proto version of Python -protobuf's benchmark. e.g. under Ubuntu, you need to +protobuf cpp implementation. You need to install the correct version of Python +C++ extension package before run generated CPP proto version of Python +protobuf's benchmark. e.g. under Ubuntu, you need to ``` -$ sudo apt-get install python-dev +$ sudo apt-get install python-dev $ sudo apt-get install python3-dev ``` -And you also need to make sure `pkg-config` is installed. +And you also need to make sure `pkg-config` is installed. ### Big data -There's some optional big testing data which is not included in the directory initially, you need to -run the following command to download the testing data: +There's some optional big testing data which is not included in the directory +initially, you need to run the following command to download the testing data: ``` -$ ./download_data.sh +$ ./download_data.sh ``` -After doing this the big data file will automaticly generated in the benchmark directory. +After doing this the big data file will automaticly generated in the +benchmark directory. ## Run instructions @@ -65,8 +66,8 @@ $ make cpp ### Python: -We have three versions of python protobuf implementation: pure python, cpp reflection and -cpp generated code. To run these version benchmark, you need to: +We have three versions of python protobuf implementation: pure python, cpp +reflection and cpp generated code. To run these version benchmark, you need to: #### Pure Python: @@ -84,8 +85,8 @@ $ make python-cpp-reflection ``` $ make python-cpp-generated-code -``` - +``` + To run a specific dataset: ### Java: diff --git a/benchmarks/cpp_benchmark.cc b/benchmarks/cpp_benchmark.cc index e26c8d11..4487643d 100644 --- a/benchmarks/cpp_benchmark.cc +++ b/benchmarks/cpp_benchmark.cc @@ -121,9 +121,10 @@ class ParseNewArenaFixture : public Fixture { virtual void BenchmarkCase(benchmark::State& state) { WrappingCounter i(payloads_.size()); size_t total = 0; + Arena arena; while (state.KeepRunning()) { - Arena arena; + arena.Reset(); Message* m = Arena::CreateMessage(&arena); const std::string& payload = payloads_[i.Next()]; total += payload.size(); diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index f82f911b..c2cd78a1 100755 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -1,15 +1,15 @@ - - + 4.0.0 protobuf-java-benchmark com.google.protobuf 1.0.0 Protocol Buffers [Benchmark] The benchmark tools for Protobuf Java. - + com.google.protobuf @@ -22,7 +22,7 @@ 1.0-beta-2 - + diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java index 88d2a83d..a7027586 100755 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java @@ -153,10 +153,8 @@ public class ProtoBenchCaliper { @Benchmark void deserializeFromByteString(int reps) throws IOException { for (int i = 0; i < reps; i++) { - defaultMessage - .newBuilderForType() - .mergeFrom(inputStringList.get((int) (counter % inputStringList.size())), extensions) - .build(); + benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( + inputStringList.get((int) (counter % inputStringList.size())), extensions); counter++; } } @@ -164,10 +162,8 @@ public class ProtoBenchCaliper { @Benchmark void deserializeFromByteArray(int reps) throws IOException { for (int i = 0; i < reps; i++) { - defaultMessage - .newBuilderForType() - .mergeFrom(inputDataList.get((int) (counter % inputDataList.size())), extensions) - .build(); + benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( + inputDataList.get((int) (counter % inputDataList.size())), extensions); counter++; } } @@ -175,10 +171,8 @@ public class ProtoBenchCaliper { @Benchmark void deserializeFromMemoryStream(int reps) throws IOException { for (int i = 0; i < reps; i++) { - defaultMessage - .newBuilderForType() - .mergeFrom(inputStreamList.get((int) (counter % inputStreamList.size())), extensions) - .build(); + benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( + inputStreamList.get((int) (counter % inputStreamList.size())), extensions); inputStreamList.get((int) (counter % inputStreamList.size())).reset(); counter++; } diff --git a/benchmarks/py_benchmark.py b/benchmarks/py_benchmark.py index d8f13cc7..d98d2a65 100755 --- a/benchmarks/py_benchmark.py +++ b/benchmarks/py_benchmark.py @@ -4,6 +4,7 @@ import timeit import math import fnmatch +# BEGIN CPP GENERATED MESSAGE # CPP generated code must be linked before importing the generated Python code # for the descriptor can be found in the pool if len(sys.argv) < 2: @@ -14,6 +15,7 @@ if sys.argv[1] == "true": sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) + "/tmp" ) elif sys.argv[1] != "false": raise IOError("Need string argument \"true\" or \"false\" for whether to use cpp generated code") +# END CPP GENERATED MESSAGE import datasets.google_message1.benchmark_message1_proto2_pb2 as benchmark_message1_proto2_pb2 import datasets.google_message1.benchmark_message1_proto3_pb2 as benchmark_message1_proto3_pb2 @@ -37,15 +39,15 @@ def run_one_test(filename): benchmark_util.set_test_method("serialize_to_benchmark") print benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) print "" - -def init(filename): + +def init(filename): global benchmark_dataset, message_class, message_list, counter message_list=[] counter = 0 data = open(os.path.dirname(sys.argv[0]) + "/../" + filename).read() benchmark_dataset = benchmarks_pb2.BenchmarkDataset() benchmark_dataset.ParseFromString(data) - + if benchmark_dataset.message_name == "benchmarks.proto3.GoogleMessage1": message_class = benchmark_message1_proto3_pb2.GoogleMessage1 elif benchmark_dataset.message_name == "benchmarks.proto2.GoogleMessage1": @@ -53,63 +55,63 @@ def init(filename): elif benchmark_dataset.message_name == "benchmarks.proto2.GoogleMessage2": message_class = benchmark_message2_pb2.GoogleMessage2 elif benchmark_dataset.message_name == "benchmarks.google_message3.GoogleMessage3": - message_class = benchmark_message3_pb2.GoogleMessage3 + message_class = benchmark_message3_pb2.GoogleMessage3 elif benchmark_dataset.message_name == "benchmarks.google_message4.GoogleMessage4": message_class = benchmark_message4_pb2.GoogleMessage4 - else: + else: raise IOError("Message %s not found!" % (benchmark_dataset.message_name)) - + for one_payload in benchmark_dataset.payload: temp = message_class() temp.ParseFromString(one_payload) message_list.append(temp) - + def parse_from_benchmark(): global counter, message_class, benchmark_dataset m = message_class().ParseFromString(benchmark_dataset.payload[counter % len(benchmark_dataset.payload)]) counter = counter + 1 - + def serialize_to_benchmark(): global counter, message_list, message_class s = message_list[counter % len(benchmark_dataset.payload)].SerializeToString() counter = counter + 1 - + class Benchmark: - def __init__(self, module=None, test_method=None, + def __init__(self, module=None, test_method=None, setup_method=None, full_iteration = 1): self.full_iteration = full_iteration self.module = module self.test_method = test_method self.setup_method = setup_method - + def set_test_method(self, test_method): self.test_method = test_method - + def full_setup_code(self, setup_method_args=''): setup_code = "" setup_code += "from %s import %s\n" % (self.module, self.test_method) setup_code += "from %s import %s\n" % (self.module, self.setup_method) setup_code += "%s(%s)\n" % (self.setup_method, setup_method_args) - return setup_code - + return setup_code + def dry_run(self, test_method_args='', setup_method_args=''): return timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args), setup=self.full_setup_code(setup_method_args), number=self.full_iteration); - + def run_benchmark(self, test_method_args='', setup_method_args=''): - reps = self.full_iteration; + reps = self.full_iteration; t = self.dry_run(test_method_args, setup_method_args); if t < 3 : - reps = int(math.ceil(3 / t)) * self.full_iteration + reps = int(math.ceil(3 / t)) * self.full_iteration t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args), setup=self.full_setup_code(setup_method_args), number=reps); return "Average time for %s: %.2f ns" % \ (self.test_method, 1.0 * t / reps * (10 ** 9)) - - + + if __name__ == "__main__": for i in range(2, len(sys.argv)): run_one_test(sys.argv[i]) -- cgit v1.2.3 From 473a8109751123eafe9ff7932703cb12b26a893a Mon Sep 17 00:00:00 2001 From: Yilun Chong Date: Wed, 10 Jan 2018 10:25:24 -0800 Subject: Update py_benchmark.py --- benchmarks/py_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/py_benchmark.py b/benchmarks/py_benchmark.py index d98d2a65..48234f03 100755 --- a/benchmarks/py_benchmark.py +++ b/benchmarks/py_benchmark.py @@ -22,7 +22,7 @@ import datasets.google_message1.benchmark_message1_proto3_pb2 as benchmark_messa import datasets.google_message2.benchmark_message2_pb2 as benchmark_message2_pb2 import datasets.google_message3.benchmark_message3_pb2 as benchmark_message3_pb2 import datasets.google_message4.benchmark_message4_pb2 as benchmark_message4_pb2 -import benchmarks_pb2 +import benchmarks_pb2 as benchmarks_pb2 def run_one_test(filename): -- cgit v1.2.3