From 195253c3f6e698fba8b58d7c810610dd86855d14 Mon Sep 17 00:00:00 2001 From: Yilun Chong Date: Fri, 5 Jan 2018 16:28:51 -0800 Subject: Add counter to Java benchmark --- .../main/java/com/google/protobuf/ProtoBench.java | 40 +++++++++++++++++++--- .../com/google/protobuf/ProtoBenchCaliper.java | 36 ++++++++++++++----- 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java index abd9200e..f8d3b797 100755 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java @@ -43,14 +43,18 @@ import com.google.protobuf.Message; import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.Scanner; public class ProtoBench { @@ -71,6 +75,7 @@ public class ProtoBench { } System.exit(success ? 0 : 1); } + /** * Runs a single test with specific test data. Error messages are displayed to stderr, @@ -86,7 +91,6 @@ public class ProtoBench { System.err.println("Unable to get input data"); return false; } - List argsList = getCaliperOption(benchmarkDataset); if (argsList == null) { System.err.println("Unable to get default message " + benchmarkDataset.getMessageName()); @@ -98,15 +102,40 @@ public class ProtoBench { try { String args[] = new String[argsList.size()]; argsList.toArray(args); - CaliperMain.exitlessMain(args, + CaliperMain.exitlessMain(args, new PrintWriter(System.out, true), new PrintWriter(System.err, true)); - return true; } catch (Exception e) { System.err.println("Error: " + e.getMessage()); System.err.println("Detailed exception information:"); e.printStackTrace(System.err); return false; } + try { + double mininumScale = 0; + // If the file not exist, this will throw IOException, which won't print the warning + // information below. + Scanner scanner = new Scanner(new String(readAllBytes("JavaBenchmarkWarning.txt"))); + while (scanner.hasNext()) { + mininumScale = Math.max(mininumScale, scanner.nextDouble()); + } + scanner.close(); + + System.out.println( + "WARNING: This benchmark's whole iterations are not enough, consider to config caliper to " + + "run for more time to make the result more convincing. You may change the configure " + + "code in com.google.protobuf.ProtoBench.getCaliperOption() of benchmark " + + benchmarkDataset.getMessageName() + + " to run for more time. e.g. Change the value of " + + "instrument.runtime.options.timingInterval or value of " + + "instrument.runtime.options.measurements to be at least " + + Math.round(mininumScale * 10 + 1) / 10.0 + + " times of before, then build and run the benchmark again\n"); + Files.deleteIfExists(Paths.get("JavaBenchmarkWarning.txt")); + } catch (IOException e) { + // The IOException here should be file not found, which means there's no warning generated by + // The benchmark, so this IOException should be discarded. + } + return true; } @@ -121,9 +150,13 @@ public class ProtoBench { } else if (benchmarkDataset.getMessageName(). equals("benchmarks.google_message3.GoogleMessage3")) { temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3"); + temp.add("-Cinstrument.runtime.options.timingInterval=3000ms"); + temp.add("-Cinstrument.runtime.options.measurements=20"); } else if (benchmarkDataset.getMessageName(). equals("benchmarks.google_message4.GoogleMessage4")) { temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4"); + temp.add("-Cinstrument.runtime.options.timingInterval=1500ms"); + temp.add("-Cinstrument.runtime.options.measurements=20"); } else { return null; } @@ -134,7 +167,6 @@ public class ProtoBench { String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream" + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream"; temp.add(benchmarkNames); - temp.add("-Cinstrument.runtime.options.timingInterval=3000ms"); return temp; } diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java index 546c25a1..88d2a83d 100755 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java @@ -2,6 +2,7 @@ package com.google.protobuf; import com.google.caliper.BeforeExperiment; +import com.google.caliper.AfterExperiment; import com.google.caliper.Benchmark; import com.google.caliper.Param; import com.google.protobuf.ByteString; @@ -11,8 +12,10 @@ import com.google.protobuf.Message; import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.BufferedWriter; import java.io.FileNotFoundException; import java.io.FileOutputStream; +import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -95,7 +98,7 @@ public class ProtoBenchCaliper { private List inputStreamList; private List inputStringList; private List sampleMessageList; - private int counter; + private long counter; @BeforeExperiment void setUp() throws IOException { @@ -125,7 +128,7 @@ public class ProtoBenchCaliper { @Benchmark void serializeToByteString(int reps) throws IOException { for (int i = 0; i < reps; i++) { - sampleMessageList.get(counter % sampleMessageList.size()).toByteString(); + sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteString(); counter++; } } @@ -133,7 +136,7 @@ public class ProtoBenchCaliper { @Benchmark void serializeToByteArray(int reps) throws IOException { for (int i = 0; i < reps; i++) { - sampleMessageList.get(counter % sampleMessageList.size()).toByteArray(); + sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteArray(); counter++; } } @@ -142,7 +145,7 @@ public class ProtoBenchCaliper { void serializeToMemoryStream(int reps) throws IOException { for (int i = 0; i < reps; i++) { ByteArrayOutputStream output = new ByteArrayOutputStream(); - sampleMessageList.get(counter % sampleMessageList.size()).writeTo(output); + sampleMessageList.get((int) (counter % sampleMessageList.size())).writeTo(output); counter++; } } @@ -152,7 +155,7 @@ public class ProtoBenchCaliper { for (int i = 0; i < reps; i++) { defaultMessage .newBuilderForType() - .mergeFrom(inputStringList.get(counter % inputStringList.size()), extensions) + .mergeFrom(inputStringList.get((int) (counter % inputStringList.size())), extensions) .build(); counter++; } @@ -163,7 +166,7 @@ public class ProtoBenchCaliper { for (int i = 0; i < reps; i++) { defaultMessage .newBuilderForType() - .mergeFrom(inputDataList.get(counter % inputDataList.size()), extensions) + .mergeFrom(inputDataList.get((int) (counter % inputDataList.size())), extensions) .build(); counter++; } @@ -174,10 +177,27 @@ public class ProtoBenchCaliper { for (int i = 0; i < reps; i++) { defaultMessage .newBuilderForType() - .mergeFrom(inputStreamList.get(counter % inputStreamList.size()), extensions) + .mergeFrom(inputStreamList.get((int) (counter % inputStreamList.size())), extensions) .build(); - inputStreamList.get(counter % inputStreamList.size()).reset(); + inputStreamList.get((int) (counter % inputStreamList.size())).reset(); counter++; } } + + @AfterExperiment + void checkCounter() throws IOException { + if (counter == 1) { + // Dry run + return; + } + if (benchmarkDataset.getPayloadCount() != 1 + && counter < benchmarkDataset.getPayloadCount() * 10L) { + BufferedWriter writer = new BufferedWriter(new FileWriter("JavaBenchmarkWarning.txt", true)); + // If the total number of non-warmup reps is smaller than 100 times of the total number of + // datasets, then output the scale that need to multiply to the configuration (either extend + // the running time for one timingInterval or run for more measurements). + writer.append(1.0 * benchmarkDataset.getPayloadCount() * 10L / counter + " "); + writer.close(); + } + } } -- cgit v1.2.3