diff options
author | Y.CORP.YAHOO.COM\tgraves <tgraves@thatenemy-lm.champ.corp.yahoo.com> | 2013-09-03 08:36:59 -0500 |
---|---|---|
committer | Y.CORP.YAHOO.COM\tgraves <tgraves@thatenemy-lm.champ.corp.yahoo.com> | 2013-09-03 08:36:59 -0500 |
commit | 547fc4a4120a2bb3a6aeca66dd0cd568534ba65a (patch) | |
tree | bfb00a0a5925ca9a08d6e7a05cc33a411a5873c1 /yarn | |
parent | 96452eea56c289e25a711ca54c812723a5059739 (diff) | |
parent | 59218bdd4996a13116009e3669b1b875be23a694 (diff) | |
download | spark-547fc4a4120a2bb3a6aeca66dd0cd568534ba65a.tar.gz spark-547fc4a4120a2bb3a6aeca66dd0cd568534ba65a.tar.bz2 spark-547fc4a4120a2bb3a6aeca66dd0cd568534ba65a.zip |
Merge remote-tracking branch 'mesos/master' into yarnUILink
Conflicts:
core/src/main/scala/org/apache/spark/ui/UIUtils.scala
core/src/main/scala/org/apache/spark/ui/jobs/PoolTable.scala
core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
docs/running-on-yarn.md
Diffstat (limited to 'yarn')
-rw-r--r-- | yarn/pom.xml | 12 | ||||
-rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala (renamed from yarn/src/main/scala/spark/deploy/yarn/ApplicationMaster.scala) | 4 | ||||
-rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala (renamed from yarn/src/main/scala/spark/deploy/yarn/ApplicationMasterArguments.scala) | 4 | ||||
-rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala (renamed from yarn/src/main/scala/spark/deploy/yarn/Client.scala) | 6 | ||||
-rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala (renamed from yarn/src/main/scala/spark/deploy/yarn/ClientArguments.scala) | 8 | ||||
-rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala (renamed from yarn/src/main/scala/spark/deploy/yarn/WorkerRunnable.scala) | 6 | ||||
-rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala (renamed from yarn/src/main/scala/spark/deploy/yarn/YarnAllocationHandler.scala) | 8 | ||||
-rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala (renamed from yarn/src/main/scala/spark/deploy/yarn/YarnSparkHadoopUtil.scala) | 4 | ||||
-rw-r--r-- | yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala (renamed from yarn/src/main/scala/spark/scheduler/cluster/YarnClusterScheduler.scala) | 6 |
9 files changed, 29 insertions, 29 deletions
diff --git a/yarn/pom.xml b/yarn/pom.xml index 07dd170eae..654b5bcd2d 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -1,5 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> -<!-- ~ Licensed to the Apache Software Foundation (ASF) under one or more +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more ~ contributor license agreements. See the NOTICE file distributed with ~ this work for additional information regarding copyright ownership. ~ The ASF licenses this file to You under the Apache License, Version 2.0 @@ -14,21 +15,20 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> - <groupId>org.spark-project</groupId> + <groupId>org.apache.spark</groupId> <artifactId>spark-parent</artifactId> <version>0.8.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <groupId>org.spark-project</groupId> + <groupId>org.apache.spark</groupId> <artifactId>spark-yarn</artifactId> <packaging>jar</packaging> <name>Spark Project YARN Support</name> - <url>http://spark-project.org/</url> + <url>http://spark.incubator.apache.org/</url> <build> <outputDirectory>target/scala-${scala.version}/classes</outputDirectory> @@ -81,7 +81,7 @@ <id>hadoop2-yarn</id> <dependencies> <dependency> - <groupId>org.spark-project</groupId> + <groupId>org.apache.spark</groupId> <artifactId>spark-core</artifactId> <version>${project.version}</version> </dependency> diff --git a/yarn/src/main/scala/spark/deploy/yarn/ApplicationMaster.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index d6acb080cc..bbeca245a8 100644 --- a/yarn/src/main/scala/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package spark.deploy.yarn +package org.apache.spark.deploy.yarn import java.net.Socket import java.util.concurrent.CopyOnWriteArrayList @@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.hadoop.yarn.ipc.YarnRPC import org.apache.hadoop.yarn.util.{ConverterUtils, Records} import scala.collection.JavaConversions._ -import spark.{SparkContext, Logging, Utils} +import org.apache.spark.{SparkContext, Logging, Utils} import org.apache.hadoop.security.UserGroupInformation import java.security.PrivilegedExceptionAction diff --git a/yarn/src/main/scala/spark/deploy/yarn/ApplicationMasterArguments.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala index 8de44b1f66..f47e23b63f 100644 --- a/yarn/src/main/scala/spark/deploy/yarn/ApplicationMasterArguments.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMasterArguments.scala @@ -15,9 +15,9 @@ * limitations under the License. */ -package spark.deploy.yarn +package org.apache.spark.deploy.yarn -import spark.util.IntParam +import org.apache.spark.util.IntParam import collection.mutable.ArrayBuffer class ApplicationMasterArguments(val args: Array[String]) { diff --git a/yarn/src/main/scala/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index eb2a8cc642..48e737ed79 100644 --- a/yarn/src/main/scala/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package spark.deploy.yarn +package org.apache.spark.deploy.yarn import java.net.{InetSocketAddress, URI} import java.nio.ByteBuffer @@ -33,10 +33,10 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.hadoop.yarn.ipc.YarnRPC import scala.collection.mutable.HashMap import scala.collection.JavaConversions._ -import spark.{Logging, Utils} +import org.apache.spark.{Logging, Utils} import org.apache.hadoop.yarn.util.{Apps, Records, ConverterUtils} import org.apache.hadoop.yarn.api.ApplicationConstants.Environment -import spark.deploy.SparkHadoopUtil +import org.apache.spark.deploy.SparkHadoopUtil class Client(conf: Configuration, args: ClientArguments) extends YarnClientImpl with Logging { diff --git a/yarn/src/main/scala/spark/deploy/yarn/ClientArguments.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala index 67aff03781..6cbfadc23b 100644 --- a/yarn/src/main/scala/spark/deploy/yarn/ClientArguments.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientArguments.scala @@ -15,12 +15,12 @@ * limitations under the License. */ -package spark.deploy.yarn +package org.apache.spark.deploy.yarn -import spark.util.MemoryParam -import spark.util.IntParam +import org.apache.spark.util.MemoryParam +import org.apache.spark.util.IntParam import collection.mutable.{ArrayBuffer, HashMap} -import spark.scheduler.{InputFormatInfo, SplitInfo} +import org.apache.spark.scheduler.{InputFormatInfo, SplitInfo} // TODO: Add code and support for ensuring that yarn resource 'asks' are location aware ! class ClientArguments(val args: Array[String]) { diff --git a/yarn/src/main/scala/spark/deploy/yarn/WorkerRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala index 0e1fd9b680..72dcf7178e 100644 --- a/yarn/src/main/scala/spark/deploy/yarn/WorkerRunnable.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package spark.deploy.yarn +package org.apache.spark.deploy.yarn import java.net.URI import java.nio.ByteBuffer @@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment import scala.collection.JavaConversions._ import scala.collection.mutable.HashMap -import spark.{Logging, Utils} +import org.apache.spark.{Logging, Utils} class WorkerRunnable(container: Container, conf: Configuration, masterAddress: String, slaveId: String, hostname: String, workerMemory: Int, workerCores: Int) @@ -119,7 +119,7 @@ class WorkerRunnable(container: Container, conf: Configuration, masterAddress: S // TODO: If the OOM is not recoverable by rescheduling it on different node, then do 'something' to fail job ... akin to blacklisting trackers in mapred ? " -XX:OnOutOfMemoryError='kill %p' " + JAVA_OPTS + - " spark.executor.StandaloneExecutorBackend " + + " org.apache.spark.executor.StandaloneExecutorBackend " + masterAddress + " " + slaveId + " " + hostname + " " + diff --git a/yarn/src/main/scala/spark/deploy/yarn/YarnAllocationHandler.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala index 1f235cef88..0a3b3abc74 100644 --- a/yarn/src/main/scala/spark/deploy/yarn/YarnAllocationHandler.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala @@ -15,13 +15,13 @@ * limitations under the License. */ -package spark.deploy.yarn +package org.apache.spark.deploy.yarn -import spark.{Logging, Utils} -import spark.scheduler.SplitInfo +import org.apache.spark.{Logging, Utils} +import org.apache.spark.scheduler.SplitInfo import scala.collection import org.apache.hadoop.yarn.api.records.{AMResponse, ApplicationAttemptId, ContainerId, Priority, Resource, ResourceRequest, ContainerStatus, Container} -import spark.scheduler.cluster.{ClusterScheduler, StandaloneSchedulerBackend} +import org.apache.spark.scheduler.cluster.{ClusterScheduler, StandaloneSchedulerBackend} import org.apache.hadoop.yarn.api.protocolrecords.{AllocateRequest, AllocateResponse} import org.apache.hadoop.yarn.util.{RackResolver, Records} import java.util.concurrent.{CopyOnWriteArrayList, ConcurrentHashMap} diff --git a/yarn/src/main/scala/spark/deploy/yarn/YarnSparkHadoopUtil.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala index 77c4ee7f3f..ca2f1e2565 100644 --- a/yarn/src/main/scala/spark/deploy/yarn/YarnSparkHadoopUtil.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala @@ -15,9 +15,9 @@ * limitations under the License. */ -package spark.deploy.yarn +package org.apache.spark.deploy.yarn -import spark.deploy.SparkHadoopUtil +import org.apache.spark.deploy.SparkHadoopUtil import collection.mutable.HashMap import org.apache.hadoop.mapred.JobConf import org.apache.hadoop.security.UserGroupInformation diff --git a/yarn/src/main/scala/spark/scheduler/cluster/YarnClusterScheduler.scala b/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala index 58a3f4043a..bbc96cfef7 100644 --- a/yarn/src/main/scala/spark/scheduler/cluster/YarnClusterScheduler.scala +++ b/yarn/src/main/scala/org/apache/spark/scheduler/cluster/YarnClusterScheduler.scala @@ -15,10 +15,10 @@ * limitations under the License. */ -package spark.scheduler.cluster +package org.apache.spark.scheduler.cluster -import spark._ -import spark.deploy.yarn.{ApplicationMaster, YarnAllocationHandler} +import org.apache.spark._ +import org.apache.spark.deploy.yarn.{ApplicationMaster, YarnAllocationHandler} import org.apache.hadoop.conf.Configuration /** |