diff options
author | Liwei Lin <lwlin7@gmail.com> | 2016-09-07 10:04:00 +0100 |
---|---|---|
committer | Sean Owen <sowen@cloudera.com> | 2016-09-07 10:04:00 +0100 |
commit | 3ce3a282c8463408f9a2db93c1748e8df8087e07 (patch) | |
tree | 7814535174f3ef7294cfd20e4dfeae28fecd4693 /sql/catalyst | |
parent | 9fccde4ff80fb0fd65a9e90eb3337965e4349de4 (diff) | |
download | spark-3ce3a282c8463408f9a2db93c1748e8df8087e07.tar.gz spark-3ce3a282c8463408f9a2db93c1748e8df8087e07.tar.bz2 spark-3ce3a282c8463408f9a2db93c1748e8df8087e07.zip |
[SPARK-17359][SQL][MLLIB] Use ArrayBuffer.+=(A) instead of ArrayBuffer.append(A) in performance critical paths
## What changes were proposed in this pull request?
We should generally use `ArrayBuffer.+=(A)` rather than `ArrayBuffer.append(A)`, because `append(A)` would involve extra boxing / unboxing.
## How was this patch tested?
N/A
Author: Liwei Lin <lwlin7@gmail.com>
Closes #14914 from lw-lin/append_to_plus_eq_v2.
Diffstat (limited to 'sql/catalyst')
4 files changed, 13 insertions, 13 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala index 644a5b28a2..f93e5736de 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSet.scala @@ -55,7 +55,7 @@ class ExpressionSet protected( protected def add(e: Expression): Unit = { if (!baseSet.contains(e.canonicalized)) { baseSet.add(e.canonicalized) - originals.append(e) + originals += e } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 28064a5cfa..f982c222af 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -596,12 +596,12 @@ class CodegenContext { // also not be too small, or it will have many function calls (for wide table), see the // results in BenchmarkWideTable. if (blockBuilder.length > 1024) { - blocks.append(blockBuilder.toString()) + blocks += blockBuilder.toString() blockBuilder.clear() } blockBuilder.append(code) } - blocks.append(blockBuilder.toString()) + blocks += blockBuilder.toString() if (blocks.length == 1) { // inline execution if only one block diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala index 493b5faf9e..7512ace188 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala @@ -59,7 +59,7 @@ class QuantileSummaries( * @param x the new observation to insert into the summary */ def insert(x: Double): QuantileSummaries = { - headSampled.append(x) + headSampled += x if (headSampled.size >= defaultHeadSize) { this.withHeadBufferInserted } else { @@ -90,7 +90,7 @@ class QuantileSummaries( val currentSample = sorted(opsIdx) // Add all the samples before the next observation. while(sampleIdx < sampled.size && sampled(sampleIdx).value <= currentSample) { - newSamples.append(sampled(sampleIdx)) + newSamples += sampled(sampleIdx) sampleIdx += 1 } @@ -104,13 +104,13 @@ class QuantileSummaries( } val tuple = Stats(currentSample, 1, delta) - newSamples.append(tuple) + newSamples += tuple opsIdx += 1 } // Add all the remaining existing samples while(sampleIdx < sampled.size) { - newSamples.append(sampled(sampleIdx)) + newSamples += sampled(sampleIdx) sampleIdx += 1 } new QuantileSummaries(compressThreshold, relativeError, newSamples.toArray, currentCount) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala index 6a188e7e55..6246380dbe 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala @@ -82,8 +82,8 @@ class TreeNodeSuite extends SparkFunSuite { val expected = Seq("+", "1", "*", "2", "-", "3", "4") val expression = Add(Literal(1), Multiply(Literal(2), Subtract(Literal(3), Literal(4)))) expression transformDown { - case b: BinaryOperator => actual.append(b.symbol); b - case l: Literal => actual.append(l.toString); l + case b: BinaryOperator => actual += b.symbol; b + case l: Literal => actual += l.toString; l } assert(expected === actual) @@ -94,8 +94,8 @@ class TreeNodeSuite extends SparkFunSuite { val expected = Seq("1", "2", "3", "4", "-", "*", "+") val expression = Add(Literal(1), Multiply(Literal(2), Subtract(Literal(3), Literal(4)))) expression transformUp { - case b: BinaryOperator => actual.append(b.symbol); b - case l: Literal => actual.append(l.toString); l + case b: BinaryOperator => actual += b.symbol; b + case l: Literal => actual += l.toString; l } assert(expected === actual) @@ -134,8 +134,8 @@ class TreeNodeSuite extends SparkFunSuite { val expected = Seq("1", "2", "3", "4", "-", "*", "+") val expression = Add(Literal(1), Multiply(Literal(2), Subtract(Literal(3), Literal(4)))) expression foreachUp { - case b: BinaryOperator => actual.append(b.symbol); - case l: Literal => actual.append(l.toString); + case b: BinaryOperator => actual += b.symbol; + case l: Literal => actual += l.toString; } assert(expected === actual) |