diff options
| author | navis.ryu <navis@apache.org> | 2015-09-18 00:43:02 -0700 |
|---|---|---|
| committer | Reynold Xin <rxin@databricks.com> | 2015-09-18 00:43:02 -0700 |
| commit | e3b5d6cb29e0f983fcc55920619e6433298955f5 (patch) | |
| tree | c7c3b35f4ae6f9ac0d6461e2e77976d4e2cd81e9 /sql | |
| parent | 74d8f7dda82c3a16348f3ff22da83203e0b7f708 (diff) | |
| download | spark-e3b5d6cb29e0f983fcc55920619e6433298955f5.tar.gz spark-e3b5d6cb29e0f983fcc55920619e6433298955f5.tar.bz2 spark-e3b5d6cb29e0f983fcc55920619e6433298955f5.zip | |
[SPARK-10684] [SQL] StructType.interpretedOrdering need not to be serialized
Kryo fails with buffer overflow even with max value (2G).
{noformat}
org.apache.spark.SparkException: Kryo serialization failed: Buffer overflow. Available: 0, required: 1
Serialization trace:
containsChild (org.apache.spark.sql.catalyst.expressions.BoundReference)
child (org.apache.spark.sql.catalyst.expressions.SortOrder)
array (scala.collection.mutable.ArraySeq)
ordering (org.apache.spark.sql.catalyst.expressions.InterpretedOrdering)
interpretedOrdering (org.apache.spark.sql.types.StructType)
schema (org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema). To avoid this, increase spark.kryoserializer.buffer.max value.
at org.apache.spark.serializer.KryoSerializerInstance.serialize(KryoSerializer.scala:263)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:240)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
{noformat}
Author: navis.ryu <navis@apache.org>
Closes #8808 from navis/SPARK-10684.
Diffstat (limited to 'sql')
| -rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala index d8968ef806..b29cf22dcb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala @@ -305,7 +305,9 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru f(this) || fields.exists(field => field.dataType.existsRecursively(f)) } - private[sql] val interpretedOrdering = InterpretedOrdering.forSchema(this.fields.map(_.dataType)) + @transient + private[sql] lazy val interpretedOrdering = + InterpretedOrdering.forSchema(this.fields.map(_.dataType)) } object StructType extends AbstractDataType { |