diff options
| author | Kousuke Saruta <sarutak@oss.nttdata.co.jp> | 2014-10-28 12:37:09 -0700 |
|---|---|---|
| committer | Andrew Or <andrew@databricks.com> | 2014-10-28 12:37:09 -0700 |
| commit | 247c529b35d73c5a690bf6512b52999daba1a938 (patch) | |
| tree | e32387b888a8ed8a6099e4a75f6b19e779b8f064 /yarn | |
| parent | 1ea3e3dc9dd942402731751089bab2fb6ae29c7b (diff) | |
| download | spark-247c529b35d73c5a690bf6512b52999daba1a938.tar.gz spark-247c529b35d73c5a690bf6512b52999daba1a938.tar.bz2 spark-247c529b35d73c5a690bf6512b52999daba1a938.zip | |
[SPARK-3657] yarn alpha YarnRMClientImpl throws NPE appMasterRequest.setTrackingUrl starting spark-shell
tgravescs reported this issue.
Following is quoted from tgravescs' report.
YarnRMClientImpl.registerApplicationMaster can throw null pointer exception when setting the trackingurl if its empty:
appMasterRequest.setTrackingUrl(new URI(uiAddress).getAuthority())
I hit this just start spark-shell without the tracking url set.
14/09/23 16:18:34 INFO yarn.YarnRMClientImpl: Connecting to ResourceManager at kryptonitered-jt1.red.ygrid.yahoo.com/98.139.154.99:8030
Exception in thread "main" java.lang.NullPointerException
at org.apache.hadoop.yarn.proto.YarnServiceProtos$RegisterApplicationMasterRequestProto$Builder.setTrackingUrl(YarnServiceProtos.java:710)
at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterRequestPBImpl.setTrackingUrl(RegisterApplicationMasterRequestPBImpl.java:132)
at org.apache.spark.deploy.yarn.YarnRMClientImpl.registerApplicationMaster(YarnRMClientImpl.scala:102)
at org.apache.spark.deploy.yarn.YarnRMClientImpl.register(YarnRMClientImpl.scala:55)
at org.apache.spark.deploy.yarn.YarnRMClientImpl.register(YarnRMClientImpl.scala:38)
at org.apache.spark.deploy.yarn.ApplicationMaster.registerAM(ApplicationMaster.scala:168)
at org.apache.spark.deploy.yarn.ApplicationMaster.runExecutorLauncher(ApplicationMaster.scala:206)
at org.apache.spark.deploy.yarn.ApplicationMaster.run(ApplicationMaster.scala:120)
Author: Kousuke Saruta <sarutak@oss.nttdata.co.jp>
Closes #2981 from sarutak/SPARK-3657-2 and squashes the following commits:
e2fd6bc [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into SPARK-3657
70b8882 [Kousuke Saruta] Fixed NPE thrown
Diffstat (limited to 'yarn')
| -rw-r--r-- | yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala index 7faf55bc63..e342cc82f4 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala @@ -18,7 +18,7 @@ package org.apache.spark.deploy.yarn import scala.collection.{Map, Set} -import java.net.URI; +import java.net.URI import org.apache.hadoop.net.NetUtils import org.apache.hadoop.yarn.api._ @@ -109,7 +109,9 @@ private class YarnRMClientImpl(args: ApplicationMasterArguments) extends YarnRMC appMasterRequest.setHost(Utils.localHostName()) appMasterRequest.setRpcPort(0) // remove the scheme from the url if it exists since Hadoop does not expect scheme - appMasterRequest.setTrackingUrl(new URI(uiAddress).getAuthority()) + val uri = new URI(uiAddress) + val authority = if (uri.getScheme == null) uiAddress else uri.getAuthority + appMasterRequest.setTrackingUrl(authority) resourceManager.registerApplicationMaster(appMasterRequest) } |