From 7f57d13c22356b491099a2884caf23a9cf672df3 Mon Sep 17 00:00:00 2001 From: Yilun Chong Date: Mon, 16 Jul 2018 13:53:22 -0700 Subject: add php benchmark --- benchmarks/Makefile.am | 56 +++++++++++++- benchmarks/README.md | 28 +++++++ benchmarks/php/PhpBenchmark.php | 157 ++++++++++++++++++++++++++++++++++++++++ benchmarks/php/autoload.php | 25 +++++++ 4 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 benchmarks/php/PhpBenchmark.php create mode 100644 benchmarks/php/autoload.php (limited to 'benchmarks') diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index 9f609228..c7e6ee20 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -36,7 +36,7 @@ protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_ oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd/cpp --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) ) touch protoc_middleman2 -all_data = $$(find $(srcdir) -type f -name "dataset.*.pb" -not -path "./tmp/*") +all_data = $$(find $$(cd $(srcdir) && pwd) -type f -name "dataset.*.pb" -not -path "$$(cd $(srcdir) && pwd)/tmp/*") ############# CPP RULES ############## @@ -491,7 +491,8 @@ proto3_proto_middleman: protoc-gen-proto2_to_proto3 oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I$(srcdir) -I$(top_srcdir) --plugin=protoc-gen-proto2_to_proto3 --proto2_to_proto3_out=$$oldpwd/tmp/proto3_proto $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) $(benchmarks_protoc_inputs_proto2) ) touch proto3_proto_middleman -proto3_data = $$(for data in $(all_data); do echo "tmp/proto3_data$${data\#$(srcdir)}"; done | xargs) +full_srcdir = $$(cd $(srcdir) && pwd) +proto3_data = $$(for data in $(all_data); do echo $(full_srcdir)"/tmp/proto3_data$${data\#$(full_srcdir)}"; done | xargs) generate_proto3_data: protoc_middleman protoc_middleman2 proto3-data-stripper mkdir -p `dirname $(proto3_data)` @@ -500,6 +501,51 @@ generate_proto3_data: protoc_middleman protoc_middleman2 proto3-data-stripper ############ PROTO3 PREPARATION END ############# +############ PHP RULES BEGIN ################# + +proto3_middleman_php: proto3_proto_middleman + mkdir -p "tmp/php" + oldpwd=`pwd` && ( cd tmp/proto3_proto && $$oldpwd/../src/protoc$(EXEEXT) -I$(srcdir) -I$(top_srcdir) --php_out=$$oldpwd/tmp/php $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) $(benchmarks_protoc_inputs_proto2) ) + touch proto3_middleman_php + +php-benchmark: proto3_middleman_php generate_proto3_data + mkdir -p "tmp/php/Google/Protobuf/Benchmark" && cp php/PhpBenchmark.php "tmp/php/Google/Protobuf/Benchmark" + cp php/autoload.php "tmp/php" + @echo "Writing shortcut script php-benchmark..." + @echo '#! /bin/bash' > php-benchmark + @echo 'export PROTOBUF_PHP_SRCDIR="$$(cd $(top_srcdir) && pwd)/php/src"' >> php-benchmark + @echo 'cd tmp/php' >> php-benchmark + @echo 'export CURRENT_DIR=$$(pwd)' >> php-benchmark + @echo 'php -d auto_prepend_file="autoload.php" -d include_path="$$(pwd)" Google/Protobuf/Benchmark/PhpBenchmark.php $$@' >> php-benchmark + @echo 'cd ../..' >> php-benchmark + @chmod +x php-benchmark + +php: php-benchmark proto3_middleman_php + ./php-benchmark $(proto3_data) + +php_c_extension: + cd $(top_srcdir)/php/ext/google/protobuf && ./configure CFLAGS='-O3' && make -j8 + +php-c-benchmark: proto3_middleman_php generate_proto3_data php_c_extension php_c_extension + mkdir -p "tmp/php/Google/Protobuf/Benchmark" && cp php/PhpBenchmark.php "tmp/php/Google/Protobuf/Benchmark" + cp php/autoload.php "tmp/php" + @echo "Writing shortcut script php-c-benchmark..." + @echo '#! /bin/bash' > php-c-benchmark + @echo 'export PROTOBUF_PHP_SRCDIR="$$(cd $(top_srcdir) && pwd)/php/src"' >> php-c-benchmark + @echo 'export PROTOBUF_PHP_EXTDIR="$$PROTOBUF_PHP_SRCDIR/../ext/google/protobuf/modules"' >> php-c-benchmark + @echo 'echo "$$PROTOBUF_PHP_EXTDIR/protobuf.so"' >> php-c-benchmark + @echo 'cd tmp/php' >> php-c-benchmark + @echo 'export CURRENT_DIR=$$(pwd)' >> php-c-benchmark + @echo 'php -d auto_prepend_file="autoload.php" -d include_path="$$(pwd)" -d extension="$$PROTOBUF_PHP_EXTDIR/protobuf.so" Google/Protobuf/Benchmark/PhpBenchmark.php $$@' >> php-c-benchmark + @echo 'cd ../..' >> php-c-benchmark + @chmod +x php-c-benchmark + +php_c: php-c-benchmark proto3_middleman_php + ./php-c-benchmark $(proto3_data) + + +############ PHP RULES END ################# + MAINTAINERCLEANFILES = \ Makefile.in @@ -544,8 +590,12 @@ CLEANFILES = \ gogo-benchmark \ gogo/cpp_no_group/cpp_benchmark.* \ proto3_proto_middleman \ - generate_proto3_data + generate_proto3_data \ + php-benchmark \ + php-c-benchmark \ + proto3_middleman_php clean-local: -rm -rf tmp/* + diff --git a/benchmarks/README.md b/benchmarks/README.md index ae5c7ddd..91b80a92 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -58,6 +58,10 @@ $ export PATH=$PATH:$(go env GOPATH)/bin The first command installs `protoc-gen-go` into the `bin` directory in your local `GOPATH`. The second command adds the `bin` directory to your `PATH` so that `protoc` can locate the plugin later. +### PHP +PHP benchmark's requirement is the same as PHP protobuf's requirements. The benchmark will automaticly +include PHP protobuf's src and build the c extension if required. + ### Big data There's some optional big testing data which is not included in the directory @@ -120,6 +124,18 @@ $ make python-cpp-generated-code $ make go ``` + +### PHP +We have two version of php protobuf implemention: pure php, php with c extension. To run these version benchmark, you need to: +#### Pure PHP +``` +$ make php +``` +#### PHP with c extension +``` +$ make php_c +``` + To run a specific dataset or run with specific options: ### Java: @@ -167,6 +183,18 @@ $ make go-benchmark $ ./go-benchmark $(specific generated dataset file name) [go testing options] ``` +### PHP +#### Pure PHP +``` +$ make php-benchmark +$ ./php-benchmark $(specific generated dataset file name) +``` +#### PHP with c extension +``` +$ make php-c-benchmark +$ ./php-c-benchmark $(specific generated dataset file name) +``` + ## Benchmark datasets diff --git a/benchmarks/php/PhpBenchmark.php b/benchmarks/php/PhpBenchmark.php new file mode 100644 index 00000000..058940d1 --- /dev/null +++ b/benchmarks/php/PhpBenchmark.php @@ -0,0 +1,157 @@ +getPayload(); + for ($i = $payloads->count() - 1; $i >= 0; $i--) { + (new $args[1]())->mergeFromString($payloads->offsetGet($i)); + } + } + + // $args: array of message + static function serialize(&$args) { + foreach ($args as &$temp_message) { + $temp_message->serializeToString(); + } + } +} + +class Benchmark +{ + private $benchmark_name; + private $args; + private $benchmark_time; + private $total_bytes; + private $coefficient; + + public function __construct($benchmark_name, $args, $total_bytes, + $benchmark_time = 5.0) { + $this->args = $args; + $this->benchmark_name = $benchmark_name; + $this->benchmark_time = $benchmark_time; + $this->total_bytes = $total_bytes; + $this->coefficient = pow (10, 0) / pow(2, 20); + } + + public function runBenchmark() { + $t = $this->runBenchmarkWithTimes(1); + $times = ceil($this->benchmark_time / $t); + return $this->total_bytes * $times / + $this->runBenchmarkWithTimes($times) * + $this->coefficient; + } + + private function runBenchmarkWithTimes($times) { + $st = microtime(true); + for ($i = 0; $i < $times; $i++) { + call_user_func_array($this->benchmark_name, array(&$this->args)); + } + $en = microtime(true); + return $en - $st; + } +} + +function getMessageName(&$dataset) { + switch ($dataset->getMessageName()) { + case "benchmarks.proto3.GoogleMessage1": + return "\Benchmarks\Proto3\GoogleMessage1"; + case "benchmarks.proto2.GoogleMessage1": + return "\Benchmarks\Proto2\GoogleMessage1"; + case "benchmarks.proto2.GoogleMessage2": + return "\Benchmarks\Proto2\GoogleMessage2"; + case "benchmarks.google_message3.GoogleMessage3": + return "\Benchmarks\Google_message3\GoogleMessage3"; + case "benchmarks.google_message4.GoogleMessage4": + return "\Benchmarks\Google_message4\GoogleMessage4"; + default: + exit("Message " . $dataset->getMessageName() . " not found !"); + } +} + +function runBenchmark($file) { + $datafile = fopen($file, "r") or die("Unable to open file " . $file); + $bytes = fread($datafile, filesize($file)); + $dataset = new BenchmarkDataset(NULL); + $dataset->mergeFromString($bytes); + $message_name = getMessageName($dataset); + $message_list = array(); + $total_bytes = 0; + $payloads = $dataset->getPayload(); + for ($i = $payloads->count() - 1; $i >= 0; $i--) { + $new_message = new $message_name(); + $new_message->mergeFromString($payloads->offsetGet($i)); + array_push($message_list, $new_message); + $total_bytes += strlen($payloads->offsetGet($i)); + } + + $parse_benchmark = new Benchmark( + "\Google\Protobuf\Benchmark\BenchmarkMethod::parse", + array($dataset, $message_name), $total_bytes); + $serialize_benchmark = new Benchmark( + "\Google\Protobuf\Benchmark\BenchmarkMethod::serialize", + $message_list, $total_bytes); + + return array( + "filename" => $file, + "benchmarks" => array( + "parse_php" => $parse_benchmark->runBenchmark(), + "serailize_php" => $serialize_benchmark->runBenchmark() + ), + "message_name" => $dataset->getMessageName() + ); +} + +// main +$json_output = false; +$results = array(); +foreach ($argv as $index => $arg) { + if ($index == 0) { + continue; + } + if ($arg == "--json") { + $json_output = true; + continue; + } else { + array_push($results, runBenchmark($arg)); + } +} + +if ($json_output) { + print json_encode($results); +} else { + print "PHP protobuf benchmark result:\n\n"; + foreach ($results as $result) { + printf("result for test data file: %s\n", $result["filename"]); + foreach ($result["benchmarks"] as $benchmark => $throughput) { + printf(" Throughput for benchmark %s: %.2f MB/s\n", + $benchmark, $throughput); + } + } +} + +?> diff --git a/benchmarks/php/autoload.php b/benchmarks/php/autoload.php new file mode 100644 index 00000000..52a8741f --- /dev/null +++ b/benchmarks/php/autoload.php @@ -0,0 +1,25 @@ +