diff options
| author | Reynold Xin <rxin@apache.org> | 2014-09-28 20:32:54 -0700 |
|---|---|---|
| committer | Reynold Xin <rxin@apache.org> | 2014-09-28 20:32:54 -0700 |
| commit | f350cd307045c2c02e713225d8f1247f18ba123e (patch) | |
| tree | 9cdd6fe5905d9ff9160ec0232b9d6079837a75e0 /sql | |
| parent | 25164a89dd32eef58d9b6823ae259439f796e81a (diff) | |
| download | spark-f350cd307045c2c02e713225d8f1247f18ba123e.tar.gz spark-f350cd307045c2c02e713225d8f1247f18ba123e.tar.bz2 spark-f350cd307045c2c02e713225d8f1247f18ba123e.zip | |
[SPARK-3543] TaskContext remaining cleanup work.
Author: Reynold Xin <rxin@apache.org>
Closes #2560 from rxin/TaskContext and squashes the following commits:
9eff95a [Reynold Xin] [SPARK-3543] remaining cleanup work.
Diffstat (limited to 'sql')
| -rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTableOperations.scala | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTableOperations.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTableOperations.scala index d39e31a7fa..ffb732347d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTableOperations.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTableOperations.scala @@ -289,9 +289,9 @@ case class InsertIntoParquetTable( def writeShard(context: TaskContext, iter: Iterator[Row]): Int = { // Hadoop wants a 32-bit task attempt ID, so if ours is bigger than Int.MaxValue, roll it // around by taking a mod. We expect that no task will be attempted 2 billion times. - val attemptNumber = (context.attemptId % Int.MaxValue).toInt + val attemptNumber = (context.getAttemptId % Int.MaxValue).toInt /* "reduce task" <split #> <attempt # = spark task #> */ - val attemptId = newTaskAttemptID(jobtrackerID, stageId, isMap = false, context.partitionId, + val attemptId = newTaskAttemptID(jobtrackerID, stageId, isMap = false, context.getPartitionId, attemptNumber) val hadoopContext = newTaskAttemptContext(wrappedConf.value, attemptId) val format = new AppendingParquetOutputFormat(taskIdOffset) |