diff options
Diffstat (limited to 'mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala')
-rw-r--r-- | mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala index f07374ac0c..ba2e1e2bc2 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala @@ -181,20 +181,20 @@ class DefaultSource extends FileFormat with DataSourceRegister { partitionSchema: StructType, requiredSchema: StructType, filters: Seq[Filter], - options: Map[String, String]): (PartitionedFile) => Iterator[InternalRow] = { + options: Map[String, String], + hadoopConf: Configuration): (PartitionedFile) => Iterator[InternalRow] = { verifySchema(dataSchema) val numFeatures = options("numFeatures").toInt assert(numFeatures > 0) val sparse = options.getOrElse("vectorType", "sparse") == "sparse" - val broadcastedConf = sparkSession.sparkContext.broadcast( - new SerializableConfiguration( - new Configuration(sparkSession.sparkContext.hadoopConfiguration))) + val broadcastedHadoopConf = + sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf)) (file: PartitionedFile) => { val points = - new HadoopFileLinesReader(file, broadcastedConf.value.value) + new HadoopFileLinesReader(file, broadcastedHadoopConf.value.value) .map(_.toString.trim) .filterNot(line => line.isEmpty || line.startsWith("#")) .map { line => |