diff options
6 files changed, 25 insertions, 20 deletions
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java b/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java index 696ee73a76..cf38a04ed7 100644 --- a/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java +++ b/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java @@ -376,7 +376,8 @@ final class ShuffleExternalSorter extends MemoryConsumer { // for tests assert(inMemSorter != null); if (inMemSorter.numRecords() >= numElementsForSpillThreshold) { - logger.info("Spilling data because number of spilledRecords crossed the threshold " + numElementsForSpillThreshold); + logger.info("Spilling data because number of spilledRecords crossed the threshold " + + numElementsForSpillThreshold); spill(); } diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java index d6a255ed9d..8d596f87d2 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java @@ -27,7 +27,6 @@ import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.spark.SparkEnv; import org.apache.spark.TaskContext; import org.apache.spark.executor.ShuffleWriteMetrics; import org.apache.spark.memory.MemoryConsumer; @@ -99,8 +98,8 @@ public final class UnsafeExternalSorter extends MemoryConsumer { long numElementsForSpillThreshold, UnsafeInMemorySorter inMemorySorter) throws IOException { UnsafeExternalSorter sorter = new UnsafeExternalSorter(taskMemoryManager, blockManager, - serializerManager, taskContext, recordComparator, prefixComparator, initialSize, numElementsForSpillThreshold, - pageSizeBytes, inMemorySorter, false /* ignored */); + serializerManager, taskContext, recordComparator, prefixComparator, initialSize, + numElementsForSpillThreshold, pageSizeBytes, inMemorySorter, false /* ignored */); sorter.spill(Long.MAX_VALUE, sorter); // The external sorter will be used to insert records, in-memory sorter is not needed. sorter.inMemSorter = null; @@ -119,8 +118,8 @@ public final class UnsafeExternalSorter extends MemoryConsumer { long numElementsForSpillThreshold, boolean canUseRadixSort) { return new UnsafeExternalSorter(taskMemoryManager, blockManager, serializerManager, - taskContext, recordComparator, prefixComparator, initialSize, pageSizeBytes, numElementsForSpillThreshold, null, - canUseRadixSort); + taskContext, recordComparator, prefixComparator, initialSize, pageSizeBytes, + numElementsForSpillThreshold, null, canUseRadixSort); } private UnsafeExternalSorter( @@ -387,7 +386,8 @@ public final class UnsafeExternalSorter extends MemoryConsumer { assert(inMemSorter != null); if (inMemSorter.numRecords() >= numElementsForSpillThreshold) { - logger.info("Spilling data because number of spilledRecords crossed the threshold " + numElementsForSpillThreshold); + logger.info("Spilling data because number of spilledRecords crossed the threshold " + + numElementsForSpillThreshold); spill(); } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/xml/UDFXPathUtil.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/xml/UDFXPathUtil.java index 01a11f9bdc..410e9e51ba 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/xml/UDFXPathUtil.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/xml/UDFXPathUtil.java @@ -19,7 +19,6 @@ package org.apache.spark.sql.catalyst.expressions.xml; import java.io.IOException; import java.io.Reader; -import java.io.StringReader; import javax.xml.namespace.QName; import javax.xml.xpath.XPath; @@ -71,7 +70,7 @@ public class UDFXPathUtil { try { return expression.evaluate(inputSource, qname); } catch (XPathExpressionException e) { - throw new RuntimeException ("Invalid expression '" + oldPath + "'", e); + throw new RuntimeException("Invalid expression '" + oldPath + "'", e); } } @@ -96,7 +95,7 @@ public class UDFXPathUtil { } /** - * Reusable, non-threadsafe version of {@link StringReader}. + * Reusable, non-threadsafe version of {@link java.io.StringReader}. */ public static class ReusableStringReader extends Reader { @@ -117,20 +116,22 @@ public class UDFXPathUtil { /** Check to make sure that the stream has not been closed */ private void ensureOpen() throws IOException { - if (str == null) + if (str == null) { throw new IOException("Stream closed"); + } } @Override public int read() throws IOException { ensureOpen(); - if (next >= length) + if (next >= length) { return -1; + } return str.charAt(next++); } @Override - public int read(char cbuf[], int off, int len) throws IOException { + public int read(char[] cbuf, int off, int len) throws IOException { ensureOpen(); if ((off < 0) || (off > cbuf.length) || (len < 0) || ((off + len) > cbuf.length) || ((off + len) < 0)) { @@ -138,8 +139,9 @@ public class UDFXPathUtil { } else if (len == 0) { return 0; } - if (next >= length) + if (next >= length) { return -1; + } int n = Math.min(length - next, len); str.getChars(next, next + n, cbuf, off); next += n; @@ -149,8 +151,9 @@ public class UDFXPathUtil { @Override public long skip(long ns) throws IOException { ensureOpen(); - if (next >= length) + if (next >= length) { return 0; + } // Bound skip by beginning and end of the source long n = Math.min(length - next, ns); n = Math.max(-next, n); diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java index b4e87c3035..c29b002a99 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java @@ -89,8 +89,8 @@ public final class UnsafeExternalRowSorter { sparkEnv.conf().getInt("spark.shuffle.sort.initialBufferSize", DEFAULT_INITIAL_SORT_BUFFER_SIZE), pageSizeBytes, - SparkEnv.get().conf().getLong("spark.shuffle.spill.numElementsForceSpillThreshold", UnsafeExternalSorter - .DEFAULT_NUM_ELEMENTS_FOR_SPILL_THRESHOLD), + SparkEnv.get().conf().getLong("spark.shuffle.spill.numElementsForceSpillThreshold", + UnsafeExternalSorter.DEFAULT_NUM_ELEMENTS_FOR_SPILL_THRESHOLD), canUseRadixSort ); } diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMap.java b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMap.java index 3705291e1f..cd521c52d1 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMap.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMap.java @@ -247,8 +247,8 @@ public final class UnsafeFixedWidthAggregationMap { SparkEnv.get().blockManager(), SparkEnv.get().serializerManager(), map.getPageSizeBytes(), - SparkEnv.get().conf().getLong("spark.shuffle.spill.numElementsForceSpillThreshold", UnsafeExternalSorter - .DEFAULT_NUM_ELEMENTS_FOR_SPILL_THRESHOLD), + SparkEnv.get().conf().getLong("spark.shuffle.spill.numElementsForceSpillThreshold", + UnsafeExternalSorter.DEFAULT_NUM_ELEMENTS_FOR_SPILL_THRESHOLD), map); } } diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java index daa948f5c6..eb105bd09a 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java @@ -57,7 +57,8 @@ public final class UnsafeKVExternalSorter { SerializerManager serializerManager, long pageSizeBytes, long numElementsForSpillThreshold) throws IOException { - this(keySchema, valueSchema, blockManager, serializerManager, pageSizeBytes, numElementsForSpillThreshold, null); + this(keySchema, valueSchema, blockManager, serializerManager, pageSizeBytes, + numElementsForSpillThreshold, null); } public UnsafeKVExternalSorter( |