diff options
| author | Reynold Xin <rxin@apache.org> | 2013-12-31 17:48:24 -0800 |
|---|---|---|
| committer | Reynold Xin <rxin@apache.org> | 2013-12-31 17:48:24 -0800 |
| commit | 8b8e70ebde880d08ebb3816b2f4003247559c7f8 (patch) | |
| tree | aa984e1263c1e825b50c80e6651a35d686bf2c7d | |
| parent | 63b411dd8664c27ac55586d8345733afad80961f (diff) | |
| parent | bee445c927586136673f39259f23642a5a6e8efe (diff) | |
| download | spark-8b8e70ebde880d08ebb3816b2f4003247559c7f8.tar.gz spark-8b8e70ebde880d08ebb3816b2f4003247559c7f8.tar.bz2 spark-8b8e70ebde880d08ebb3816b2f4003247559c7f8.zip | |
Merge pull request #73 from falaki/ApproximateDistinctCount
Approximate distinct count
Added countApproxDistinct() to RDD and countApproxDistinctByKey() to PairRDDFunctions to approximately count distinct number of elements and distinct number of values per key, respectively. Both functions use HyperLogLog from stream-lib for counting. Both functions take a parameter that controls the trade-off between accuracy and memory consumption. Also added Scala docs and test suites for both methods.
| -rw-r--r-- | core/pom.xml | 1582 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala | 36 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala | 11 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 42 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/rdd/RDD.scala | 16 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/util/SerializableHyperLogLog.scala | 50 | ||||
| -rw-r--r-- | core/src/test/scala/org/apache/spark/JavaAPISuite.java | 32 | ||||
| -rw-r--r-- | core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala | 34 | ||||
| -rw-r--r-- | core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala | 13 | ||||
| -rw-r--r-- | core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala | 4 | ||||
| -rw-r--r-- | pom.xml | 5 | ||||
| -rw-r--r-- | project/SparkBuild.scala | 3 |
12 files changed, 1595 insertions, 233 deletions
diff --git a/core/pom.xml b/core/pom.xml index 043f6cf68d..f0248bc539 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -1,231 +1,1351 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one or more - ~ contributor license agreements. See the NOTICE file distributed with - ~ this work for additional information regarding copyright ownership. - ~ The ASF licenses this file to You under the Apache License, Version 2.0 - ~ (the "License"); you may not use this file except in compliance with - ~ the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.spark</groupId> - <artifactId>spark-parent</artifactId> - <version>0.9.0-incubating-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - - <groupId>org.apache.spark</groupId> - <artifactId>spark-core_2.10</artifactId> - <packaging>jar</packaging> - <name>Spark Project Core</name> - <url>http://spark.incubator.apache.org/</url> - - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - </dependency> - <dependency> - <groupId>net.java.dev.jets3t</groupId> - <artifactId>jets3t</artifactId> - </dependency> - <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>avro</artifactId> - </dependency> - <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>avro-ipc</artifactId> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-server</artifactId> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </dependency> - <dependency> - <groupId>com.ning</groupId> - <artifactId>compress-lzf</artifactId> - </dependency> - <dependency> - <groupId>org.xerial.snappy</groupId> - <artifactId>snappy-java</artifactId> - </dependency> - <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm</artifactId> - </dependency> - <dependency> - <groupId>com.twitter</groupId> - <artifactId>chill_${scala.binary.version}</artifactId> - <version>0.3.1</version> - </dependency> - <dependency> - <groupId>com.twitter</groupId> - <artifactId>chill-java</artifactId> - <version>0.3.1</version> - </dependency> - <dependency> - <groupId>${akka.group}</groupId> - <artifactId>akka-remote_${scala.binary.version}</artifactId> - </dependency> - <dependency> - <groupId>${akka.group}</groupId> - <artifactId>akka-slf4j_${scala.binary.version}</artifactId> - </dependency> - <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-library</artifactId> - </dependency> - <dependency> - <groupId>net.liftweb</groupId> - <artifactId>lift-json_${scala.binary.version}</artifactId> - </dependency> - <dependency> - <groupId>it.unimi.dsi</groupId> - <artifactId>fastutil</artifactId> - </dependency> - <dependency> - <groupId>colt</groupId> - <artifactId>colt</artifactId> - </dependency> - <dependency> - <groupId>org.apache.mesos</groupId> - <artifactId>mesos</artifactId> - </dependency> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty-all</artifactId> - </dependency> - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </dependency> - <dependency> - <groupId>com.codahale.metrics</groupId> - <artifactId>metrics-core</artifactId> - </dependency> - <dependency> - <groupId>com.codahale.metrics</groupId> - <artifactId>metrics-jvm</artifactId> - </dependency> - <dependency> - <groupId>com.codahale.metrics</groupId> - <artifactId>metrics-json</artifactId> - </dependency> - <dependency> - <groupId>com.codahale.metrics</groupId> - <artifactId>metrics-ganglia</artifactId> - </dependency> - <dependency> - <groupId>com.codahale.metrics</groupId> - <artifactId>metrics-graphite</artifactId> - </dependency> - <dependency> - <groupId>org.apache.derby</groupId> - <artifactId>derby</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.binary.version}</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.scalacheck</groupId> - <artifactId>scalacheck_${scala.binary.version}</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.easymock</groupId> - <artifactId>easymock</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> - <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> - <execution> - <phase>test</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <exportAntProperties>true</exportAntProperties> - <tasks> - <property name="spark.classpath" refid="maven.test.classpath" /> - <property environment="env" /> - <fail message="Please set the SCALA_HOME (or SCALA_LIBRARY_PATH if scala is on the path) environment variables and retry."> - <condition> - <not> - <or> - <isset property="env.SCALA_HOME" /> - <isset property="env.SCALA_LIBRARY_PATH" /> - </or> - </not> - </condition> - </fail> - </tasks> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.scalatest</groupId> - <artifactId>scalatest-maven-plugin</artifactId> - <configuration> - <environmentVariables> - <SPARK_HOME>${basedir}/..</SPARK_HOME> - <SPARK_TESTING>1</SPARK_TESTING> - <SPARK_CLASSPATH>${spark.classpath}</SPARK_CLASSPATH> - </environmentVariables> - </configuration> - </plugin> - </plugins> - </build> -</project> + + + +<!DOCTYPE html> +<html> + <head prefix="og: http://ogp.me/ns# fb: http://ogp.me/ns/fb# githubog: http://ogp.me/ns/fb/githubog#"> + <meta charset='utf-8'> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <title>incubator-spark/core/pom.xml at master · apache/incubator-spark · GitHub</title> + <link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="GitHub" /> + <link rel="fluid-icon" href="https://github.com/fluidicon.png" title="GitHub" /> + <link rel="apple-touch-icon" sizes="57x57" href="/apple-touch-icon-114.png" /> + <link rel="apple-touch-icon" sizes="114x114" href="/apple-touch-icon-114.png" /> + <link rel="apple-touch-icon" sizes="72x72" href="/apple-touch-icon-144.png" /> + <link rel="apple-touch-icon" sizes="144x144" href="/apple-touch-icon-144.png" /> + <link rel="logo" type="image/svg" href="https://github-media-downloads.s3.amazonaws.com/github-logo.svg" /> + <meta property="og:image" content="https://github.global.ssl.fastly.net/images/modules/logos_page/Octocat.png"> + <meta name="hostname" content="github-fe126-cp1-prd.iad.github.net"> + <meta name="ruby" content="ruby 1.9.3p194-tcs-github-tcmalloc (e1c0c3f392) [x86_64-linux]"> + <link rel="assets" href="https://github.global.ssl.fastly.net/"> + <link rel="conduit-xhr" href="https://ghconduit.com:25035/"> + <link rel="xhr-socket" href="/_sockets" /> + + + + <meta name="msapplication-TileImage" content="/windows-tile.png" /> + <meta name="msapplication-TileColor" content="#ffffff" /> + <meta name="selected-link" value="repo_source" data-pjax-transient /> + <meta content="collector.githubapp.com" name="octolytics-host" /><meta content="collector-cdn.github.com" name="octolytics-script-host" /><meta content="github" name="octolytics-app-id" /><meta content="43A45EED:3C36:63F899:52C1FB77" name="octolytics-dimension-request_id" /> + + + + + <link rel="icon" type="image/x-icon" href="/favicon.ico" /> + + <meta content="authenticity_token" name="csrf-param" /> +<meta content="XL0k65gLgVyHEQhufVTLBNEWwGhZf67623b0mJZcY2A=" name="csrf-token" /> + + <link href="https://github.global.ssl.fastly.net/assets/github-3944f96c1c19f752fe766b332fb7716555c8296e.css" media="all" rel="stylesheet" type="text/css" /> + <link href="https://github.global.ssl.fastly.net/assets/github2-b64d0ad5fa62a30a166145ae08b8c0a6d2f7dea7.css" media="all" rel="stylesheet" type="text/css" /> + + + + + <script src="https://github.global.ssl.fastly.net/assets/frameworks-29a3fb0547e33bd8d4530bbad9bae3ef00d83293.js" type="text/javascript"></script> + <script src="https://github.global.ssl.fastly.net/assets/github-3fbe2841590c916eeba07af3fc626dd593d2f5ba.js" type="text/javascript"></script> + + <meta http-equiv="x-pjax-version" content="8983adfc0294e4e53e92b27093d9e927"> + + <link data-pjax-transient rel='permalink' href='/apache/incubator-spark/blob/50e3b8ec4c8150f1cfc6b92f8871f520adf2cfda/core/pom.xml'> + <meta property="og:title" content="incubator-spark"/> + <meta property="og:type" content="githubog:gitrepository"/> + <meta property="og:url" content="https://github.com/apache/incubator-spark"/> + <meta property="og:image" content="https://github.global.ssl.fastly.net/images/gravatars/gravatar-user-420.png"/> + <meta property="og:site_name" content="GitHub"/> + <meta property="og:description" content="incubator-spark - Mirror of Apache Spark"/> + + <meta name="description" content="incubator-spark - Mirror of Apache Spark" /> + + <meta content="47359" name="octolytics-dimension-user_id" /><meta content="apache" name="octolytics-dimension-user_login" /><meta content="10960835" name="octolytics-dimension-repository_id" /><meta content="apache/incubator-spark" name="octolytics-dimension-repository_nwo" /><meta content="true" name="octolytics-dimension-repository_public" /><meta content="false" name="octolytics-dimension-repository_is_fork" /><meta content="10960835" name="octolytics-dimension-repository_network_root_id" /><meta content="apache/incubator-spark" name="octolytics-dimension-repository_network_root_nwo" /> + <link href="https://github.com/apache/incubator-spark/commits/master.atom" rel="alternate" title="Recent Commits to incubator-spark:master" type="application/atom+xml" /> + + </head> + + + <body class="logged_out env-production vis-public mirror page-blob"> + <div class="wrapper"> + + + + + + + + <div class="header header-logged-out"> + <div class="container clearfix"> + + <a class="header-logo-wordmark" href="https://github.com/"> + <span class="mega-octicon octicon-logo-github"></span> + </a> + + <div class="header-actions"> + <a class="button primary" href="/join">Sign up</a> + <a class="button signin" href="/login?return_to=%2Fapache%2Fincubator-spark%2Fblob%2Fmaster%2Fcore%2Fpom.xml">Sign in</a> + </div> + + <div class="command-bar js-command-bar in-repository"> + + <ul class="top-nav"> + <li class="explore"><a href="/explore">Explore</a></li> + <li class="features"><a href="/features">Features</a></li> + <li class="enterprise"><a href="https://enterprise.github.com/">Enterprise</a></li> + <li class="blog"><a href="/blog">Blog</a></li> + </ul> + <form accept-charset="UTF-8" action="/search" class="command-bar-form" id="top_search_form" method="get"> + +<input type="text" data-hotkey="/ s" name="q" id="js-command-bar-field" placeholder="Search or type a command" tabindex="1" autocapitalize="off" + + + data-repo="apache/incubator-spark" + data-branch="master" + data-sha="ede8c631c6d7a940c1ab1629574ec1003eb0861e" + > + + <input type="hidden" name="nwo" value="apache/incubator-spark" /> + + <div class="select-menu js-menu-container js-select-menu search-context-select-menu"> + <span class="minibutton select-menu-button js-menu-target"> + <span class="js-select-button">This repository</span> + </span> + + <div class="select-menu-modal-holder js-menu-content js-navigation-container"> + <div class="select-menu-modal"> + + <div class="select-menu-item js-navigation-item js-this-repository-navigation-item selected"> + <span class="select-menu-item-icon octicon octicon-check"></span> + <input type="radio" class="js-search-this-repository" name="search_target" value="repository" checked="checked" /> + <div class="select-menu-item-text js-select-button-text">This repository</div> + </div> <!-- /.select-menu-item --> + + <div class="select-menu-item js-navigation-item js-all-repositories-navigation-item"> + <span class="select-menu-item-icon octicon octicon-check"></span> + <input type="radio" name="search_target" value="global" /> + <div class="select-menu-item-text js-select-button-text">All repositories</div> + </div> <!-- /.select-menu-item --> + + </div> + </div> + </div> + + <span class="octicon help tooltipped downwards" title="Show command bar help"> + <span class="octicon octicon-question"></span> + </span> + + + <input type="hidden" name="ref" value="cmdform"> + +</form> + </div> + + </div> +</div> + + + + + + <div class="site" itemscope itemtype="http://schema.org/WebPage"> + + <div class="pagehead repohead instapaper_ignore readability-menu"> + <div class="container"> + + +<ul class="pagehead-actions"> + + + <li> + <a href="/login?return_to=%2Fapache%2Fincubator-spark" + class="minibutton with-count js-toggler-target star-button tooltipped upwards" + title="You must be signed in to use this feature" rel="nofollow"> + <span class="octicon octicon-star"></span>Star + </a> + + <a class="social-count js-social-count" href="/apache/incubator-spark/stargazers"> + 322 + </a> + + </li> + + <li> + <a href="/login?return_to=%2Fapache%2Fincubator-spark" + class="minibutton with-count js-toggler-target fork-button tooltipped upwards" + title="You must be signed in to fork a repository" rel="nofollow"> + <span class="octicon octicon-git-branch"></span>Fork + </a> + <a href="/apache/incubator-spark/network" class="social-count"> + 273 + </a> + </li> +</ul> + + <h1 itemscope itemtype="http://data-vocabulary.org/Breadcrumb" class="entry-title public"> + <span class="repo-label"><span>public</span></span> + <span class="mega-octicon octicon-repo"></span> + <span class="author"> + <a href="/apache" class="url fn" itemprop="url" rel="author"><span itemprop="title">apache</span></a> + </span> + <span class="repohead-name-divider">/</span> + <strong><a href="/apache/incubator-spark" class="js-current-repository js-repo-home-link">incubator-spark</a></strong> + + <span class="page-context-loader"> + <img alt="Octocat-spinner-32" height="16" src="https://github.global.ssl.fastly.net/images/spinners/octocat-spinner-32.gif" width="16" /> + </span> + + <span class="mirror-flag"> + <span class="text">mirrored from <a href="git://git.apache.org/incubator-spark.git">git://git.apache.org/incubator-spark.git</a></span> + </span> + </h1> + </div><!-- /.container --> + </div><!-- /.repohead --> + + <div class="container"> + + + <div class="repository-with-sidebar repo-container "> + + <div class="repository-sidebar"> + + +<div class="sunken-menu vertical-right repo-nav js-repo-nav js-repository-container-pjax js-octicon-loaders"> + <div class="sunken-menu-contents"> + <ul class="sunken-menu-group"> + <li class="tooltipped leftwards" title="Code"> + <a href="/apache/incubator-spark" aria-label="Code" class="selected js-selected-navigation-item sunken-menu-item" data-gotokey="c" data-pjax="true" data-selected-links="repo_source repo_downloads repo_commits repo_tags repo_branches /apache/incubator-spark"> + <span class="octicon octicon-code"></span> <span class="full-word">Code</span> + <img alt="Octocat-spinner-32" class="mini-loader" height="16" src="https://github.global.ssl.fastly.net/images/spinners/octocat-spinner-32.gif" width="16" /> +</a> </li> + + + <li class="tooltipped leftwards" title="Pull Requests"> + <a href="/apache/incubator-spark/pulls" aria-label="Pull Requests" class="js-selected-navigation-item sunken-menu-item js-disable-pjax" data-gotokey="p" data-selected-links="repo_pulls /apache/incubator-spark/pulls"> + <span class="octicon octicon-git-pull-request"></span> <span class="full-word">Pull Requests</span> + <span class='counter'>45</span> + <img alt="Octocat-spinner-32" class="mini-loader" height="16" src="https://github.global.ssl.fastly.net/images/spinners/octocat-spinner-32.gif" width="16" /> +</a> </li> + + + </ul> + <div class="sunken-menu-separator"></div> + <ul class="sunken-menu-group"> + + <li class="tooltipped leftwards" title="Pulse"> + <a href="/apache/incubator-spark/pulse" aria-label="Pulse" class="js-selected-navigation-item sunken-menu-item" data-pjax="true" data-selected-links="pulse /apache/incubator-spark/pulse"> + <span class="octicon octicon-pulse"></span> <span class="full-word">Pulse</span> + <img alt="Octocat-spinner-32" class="mini-loader" height="16" src="https://github.global.ssl.fastly.net/images/spinners/octocat-spinner-32.gif" width="16" /> +</a> </li> + + <li class="tooltipped leftwards" title="Graphs"> + <a href="/apache/incubator-spark/graphs" aria-label="Graphs" class="js-selected-navigation-item sunken-menu-item" data-pjax="true" data-selected-links="repo_graphs repo_contributors /apache/incubator-spark/graphs"> + <span class="octicon octicon-graph"></span> <span class="full-word">Graphs</span> + <img alt="Octocat-spinner-32" class="mini-loader" height="16" src="https://github.global.ssl.fastly.net/images/spinners/octocat-spinner-32.gif" width="16" /> +</a> </li> + + <li class="tooltipped leftwards" title="Network"> + <a href="/apache/incubator-spark/network" aria-label="Network" class="js-selected-navigation-item sunken-menu-item js-disable-pjax" data-selected-links="repo_network /apache/incubator-spark/network"> + <span class="octicon octicon-git-branch"></span> <span class="full-word">Network</span> + <img alt="Octocat-spinner-32" class="mini-loader" height="16" src="https://github.global.ssl.fastly.net/images/spinners/octocat-spinner-32.gif" width="16" /> +</a> </li> + </ul> + + + </div> +</div> + + <div class="only-with-full-nav"> + + + + +<div class="clone-url open" + data-protocol-type="http" + data-url="/users/set_protocol?protocol_selector=http&protocol_type=clone"> + <h3><strong>HTTPS</strong> clone URL</h3> + <div class="clone-url-box"> + <input type="text" class="clone js-url-field" + value="https://github.com/apache/incubator-spark.git" readonly="readonly"> + + <span class="js-zeroclipboard url-box-clippy minibutton zeroclipboard-button" data-clipboard-text="https://github.com/apache/incubator-spark.git" data-copied-hint="copied!" title="copy to clipboard"><span class="octicon octicon-clippy"></span></span> + </div> +</div> + + + +<div class="clone-url " + data-protocol-type="subversion" + data-url="/users/set_protocol?protocol_selector=subversion&protocol_type=clone"> + <h3><strong>Subversion</strong> checkout URL</h3> + <div class="clone-url-box"> + <input type="text" class="clone js-url-field" + value="https://github.com/apache/incubator-spark" readonly="readonly"> + + <span class="js-zeroclipboard url-box-clippy minibutton zeroclipboard-button" data-clipboard-text="https://github.com/apache/incubator-spark" data-copied-hint="copied!" title="copy to clipboard"><span class="octicon octicon-clippy"></span></span> + </div> +</div> + + +<p class="clone-options">You can clone with + <a href="#" class="js-clone-selector" data-protocol="http">HTTPS</a>, + or <a href="#" class="js-clone-selector" data-protocol="subversion">Subversion</a>. + <span class="octicon help tooltipped upwards" title="Get help on which URL is right for you."> + <a href="https://help.github.com/articles/which-remote-url-should-i-use"> + <span class="octicon octicon-question"></span> + </a> + </span> +</p> + + + + <a href="/apache/incubator-spark/archive/master.zip" + class="minibutton sidebar-button" + title="Download this repository as a zip file" + rel="nofollow"> + <span class="octicon octicon-cloud-download"></span> + Download ZIP + </a> + </div> + </div><!-- /.repository-sidebar --> + + <div id="js-repo-pjax-container" class="repository-content context-loader-container" data-pjax-container> + + + +<!-- blob contrib key: blob_contributors:v21:ca1fa3336589a56eafe4ec1105b40975 --> + +<p title="This is a placeholder element" class="js-history-link-replace hidden"></p> + +<a href="/apache/incubator-spark/find/master" data-pjax data-hotkey="t" class="js-show-file-finder" style="display:none">Show File Finder</a> + +<div class="file-navigation"> + + +<div class="select-menu js-menu-container js-select-menu" > + <span class="minibutton select-menu-button js-menu-target" data-hotkey="w" + data-master-branch="master" + data-ref="master" + role="button" aria-label="Switch branches or tags" tabindex="0"> + <span class="octicon octicon-git-branch"></span> + <i>branch:</i> + <span class="js-select-button">master</span> + </span> + + <div class="select-menu-modal-holder js-menu-content js-navigation-container" data-pjax> + + <div class="select-menu-modal"> + <div class="select-menu-header"> + <span class="select-menu-title">Switch branches/tags</span> + <span class="octicon octicon-remove-close js-menu-close"></span> + </div> <!-- /.select-menu-header --> + + <div class="select-menu-filters"> + <div class="select-menu-text-filter"> + <input type="text" aria-label="Filter branches/tags" id="context-commitish-filter-field" class="js-filterable-field js-navigation-enable" placeholder="Filter branches/tags"> + </div> + <div class="select-menu-tabs"> + <ul> + <li class="select-menu-tab"> + <a href="#" data-tab-filter="branches" class="js-select-menu-tab">Branches</a> + </li> + <li class="select-menu-tab"> + <a href="#" data-tab-filter="tags" class="js-select-menu-tab">Tags</a> + </li> + </ul> + </div><!-- /.select-menu-tabs --> + </div><!-- /.select-menu-filters --> + + <div class="select-menu-list select-menu-tab-bucket js-select-menu-tab-bucket" data-tab-filter="branches"> + + <div data-filterable-for="context-commitish-filter-field" data-filterable-type="substring"> + + + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/akka-actors/core/pom.xml" + data-name="akka-actors" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="akka-actors">akka-actors</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/arthur/core/pom.xml" + data-name="arthur" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="arthur">arthur</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/branch-0.5/core/pom.xml" + data-name="branch-0.5" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="branch-0.5">branch-0.5</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/branch-0.6/core/pom.xml" + data-name="branch-0.6" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="branch-0.6">branch-0.6</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/branch-0.7/core/pom.xml" + data-name="branch-0.7" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="branch-0.7">branch-0.7</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/branch-0.8/core/pom.xml" + data-name="branch-0.8" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="branch-0.8">branch-0.8</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/charles-newhadoop/core/pom.xml" + data-name="charles-newhadoop" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="charles-newhadoop">charles-newhadoop</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/dev/core/pom.xml" + data-name="dev" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="dev">dev</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/formatting/core/pom.xml" + data-name="formatting" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="formatting">formatting</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/hive/core/pom.xml" + data-name="hive" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="hive">hive</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/java-api/core/pom.xml" + data-name="java-api" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="java-api">java-api</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item selected"> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/master/core/pom.xml" + data-name="master" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="master">master</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/mesos-0.9/core/pom.xml" + data-name="mesos-0.9" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="mesos-0.9">mesos-0.9</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/mos-bt/core/pom.xml" + data-name="mos-bt" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="mos-bt">mos-bt</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/mos-bt-dev/core/pom.xml" + data-name="mos-bt-dev" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="mos-bt-dev">mos-bt-dev</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/mos-bt-topo/core/pom.xml" + data-name="mos-bt-topo" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="mos-bt-topo">mos-bt-topo</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/mos-shuffle/core/pom.xml" + data-name="mos-shuffle" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="mos-shuffle">mos-shuffle</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/mos-shuffle-supertracked/core/pom.xml" + data-name="mos-shuffle-supertracked" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="mos-shuffle-supertracked">mos-shuffle-supertracked</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/mos-shuffle-tracked/core/pom.xml" + data-name="mos-shuffle-tracked" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="mos-shuffle-tracked">mos-shuffle-tracked</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/multi-tracker/core/pom.xml" + data-name="multi-tracker" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="multi-tracker">multi-tracker</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/new-rdds-protobuf/core/pom.xml" + data-name="new-rdds-protobuf" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="new-rdds-protobuf">new-rdds-protobuf</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/object-file-fix/core/pom.xml" + data-name="object-file-fix" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="object-file-fix">object-file-fix</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/old-mesos/core/pom.xml" + data-name="old-mesos" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="old-mesos">old-mesos</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/old-rdds/core/pom.xml" + data-name="old-rdds" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="old-rdds">old-rdds</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/perf/core/pom.xml" + data-name="perf" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="perf">perf</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/reduce-logging/core/pom.xml" + data-name="reduce-logging" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="reduce-logging">reduce-logging</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/rxin/core/pom.xml" + data-name="rxin" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="rxin">rxin</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/scala-2.8/core/pom.xml" + data-name="scala-2.8" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="scala-2.8">scala-2.8</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/scala-2.9/core/pom.xml" + data-name="scala-2.9" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="scala-2.9">scala-2.9</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/scala-2.10/core/pom.xml" + data-name="scala-2.10" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="scala-2.10">scala-2.10</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/sched-refactoring/core/pom.xml" + data-name="sched-refactoring" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="sched-refactoring">sched-refactoring</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/shuffle-fix/core/pom.xml" + data-name="shuffle-fix" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="shuffle-fix">shuffle-fix</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/sparkplug/core/pom.xml" + data-name="sparkplug" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="sparkplug">sparkplug</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/streaming/core/pom.xml" + data-name="streaming" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="streaming">streaming</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/td-checksum/core/pom.xml" + data-name="td-checksum" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="td-checksum">td-checksum</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/td-rdd-save/core/pom.xml" + data-name="td-rdd-save" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="td-rdd-save">td-rdd-save</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/tjhunter-branch/core/pom.xml" + data-name="tjhunter-branch" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="tjhunter-branch">tjhunter-branch</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/twitter-mesos/core/pom.xml" + data-name="twitter-mesos" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="twitter-mesos">twitter-mesos</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/blob/yarn/core/pom.xml" + data-name="yarn" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="yarn">yarn</a> + </div> <!-- /.select-menu-item --> + </div> + + <div class="select-menu-no-results">Nothing to show</div> + </div> <!-- /.select-menu-list --> + + <div class="select-menu-list select-menu-tab-bucket js-select-menu-tab-bucket" data-tab-filter="tags"> + <div data-filterable-for="context-commitish-filter-field" data-filterable-type="substring"> + + + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.8.1-incubating/core/pom.xml" + data-name="v0.8.1-incubating" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.8.1-incubating">v0.8.1-incubating</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.8.0-incubating-rc3/core/pom.xml" + data-name="v0.8.0-incubating-rc3" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.8.0-incubating-rc3">v0.8.0-incubating-rc3</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.8.0-incubating/core/pom.xml" + data-name="v0.8.0-incubating" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.8.0-incubating">v0.8.0-incubating</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.7.2/core/pom.xml" + data-name="v0.7.2" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.7.2">v0.7.2</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.7.1/core/pom.xml" + data-name="v0.7.1" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.7.1">v0.7.1</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.7.0-bizo-1/core/pom.xml" + data-name="v0.7.0-bizo-1" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.7.0-bizo-1">v0.7.0-bizo-1</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.7.0/core/pom.xml" + data-name="v0.7.0" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.7.0">v0.7.0</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.6.2/core/pom.xml" + data-name="v0.6.2" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.6.2">v0.6.2</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.6.1/core/pom.xml" + data-name="v0.6.1" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.6.1">v0.6.1</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.6.0-yarn/core/pom.xml" + data-name="v0.6.0-yarn" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.6.0-yarn">v0.6.0-yarn</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.6.0/core/pom.xml" + data-name="v0.6.0" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.6.0">v0.6.0</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.5.2/core/pom.xml" + data-name="v0.5.2" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.5.2">v0.5.2</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.5.1/core/pom.xml" + data-name="v0.5.1" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.5.1">v0.5.1</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/v0.5.0/core/pom.xml" + data-name="v0.5.0" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="v0.5.0">v0.5.0</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/spark-parent-0.8.1-incubating/core/pom.xml" + data-name="spark-parent-0.8.1-incubating" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="spark-parent-0.8.1-incubating">spark-parent-0.8.1-incubating</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/alpha-0.2/core/pom.xml" + data-name="alpha-0.2" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="alpha-0.2">alpha-0.2</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/alpha-0.1/core/pom.xml" + data-name="alpha-0.1" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="alpha-0.1">alpha-0.1</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/0.3-scala-2.9/core/pom.xml" + data-name="0.3-scala-2.9" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="0.3-scala-2.9">0.3-scala-2.9</a> + </div> <!-- /.select-menu-item --> + <div class="select-menu-item js-navigation-item "> + <span class="select-menu-item-icon octicon octicon-check"></span> + <a href="/apache/incubator-spark/tree/0.3-scala-2.8/core/pom.xml" + data-name="0.3-scala-2.8" + data-skip-pjax="true" + rel="nofollow" + class="js-navigation-open select-menu-item-text js-select-button-text css-truncate-target" + title="0.3-scala-2.8">0.3-scala-2.8</a> + </div> <!-- /.select-menu-item --> + </div> + + <div class="select-menu-no-results">Nothing to show</div> + </div> <!-- /.select-menu-list --> + + </div> <!-- /.select-menu-modal --> + </div> <!-- /.select-menu-modal-holder --> +</div> <!-- /.select-menu --> + + <div class="breadcrumb"> + <span class='repo-root js-repo-root'><span itemscope="" itemtype="http://data-vocabulary.org/Breadcrumb"><a href="/apache/incubator-spark" data-branch="master" data-direction="back" data-pjax="true" itemscope="url"><span itemprop="title">incubator-spark</span></a></span></span><span class="separator"> / </span><span itemscope="" itemtype="http://data-vocabulary.org/Breadcrumb"><a href="/apache/incubator-spark/tree/master/core" data-branch="master" data-direction="back" data-pjax="true" itemscope="url"><span itemprop="title">core</span></a></span><span class="separator"> / </span><strong class="final-path">pom.xml</strong> <span class="js-zeroclipboard minibutton zeroclipboard-button" data-clipboard-text="core/pom.xml" data-copied-hint="copied!" title="copy to clipboard"><span class="octicon octicon-clippy"></span></span> + </div> +</div> + + + + <div class="commit file-history-tease"> + <img class="main-avatar" height="24" src="https://2.gravatar.com/avatar/def597cee3897ba290da26972ff628d2?d=https%3A%2F%2Fidenticons.github.com%2F0ec10d5e3ed3720a2d578417a894cf49.png&r=x&s=140" width="24" /> + <span class="author"><a href="/pwendell" rel="author">pwendell</a></span> + <time class="js-relative-date" datetime="2013-12-16T21:56:21-08:00" title="2013-12-16 21:56:21">December 16, 2013</time> + <div class="commit-title"> + <a href="/apache/incubator-spark/commit/c1fec89895f03dbdbb6f445ea3cdcd2d050555c4" class="message" data-pjax="true" title="Cleanup">Cleanup</a> + </div> + + <div class="participation"> + <p class="quickstat"><a href="#blob_contributors_box" rel="facebox"><strong>15</strong> contributors</a></p> + <a class="avatar tooltipped downwards" title="ScrapCodes" href="/apache/incubator-spark/commits/master/core/pom.xml?author=ScrapCodes"><img height="20" src="https://1.gravatar.com/avatar/e9813bbbab2caa993bf7e2b2d60de894?d=https%3A%2F%2Fidenticons.github.com%2F38c660c74f82a216b75167debab770ed.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="mateiz" href="/apache/incubator-spark/commits/master/core/pom.xml?author=mateiz"><img height="20" src="https://2.gravatar.com/avatar/17d3d634fc898bf3ae19444450af6805?d=https%3A%2F%2Fidenticons.github.com%2F627fcfc1b5e8e4535cc26ecfa133743e.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="jey" href="/apache/incubator-spark/commits/master/core/pom.xml?author=jey"><img height="20" src="https://1.gravatar.com/avatar/85a3f16c85ee6e38527e82e035ed4bf9?d=https%3A%2F%2Fidenticons.github.com%2F2ebb6c06bdc16ef37ec965c6b325b5c6.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="markhamstra" href="/apache/incubator-spark/commits/master/core/pom.xml?author=markhamstra"><img height="20" src="https://2.gravatar.com/avatar/aaa2e907159f6919c4a4c8039d46752f?d=https%3A%2F%2Fidenticons.github.com%2F80aa657180765c73a93528281452d8dc.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="pwendell" href="/apache/incubator-spark/commits/master/core/pom.xml?author=pwendell"><img height="20" src="https://2.gravatar.com/avatar/def597cee3897ba290da26972ff628d2?d=https%3A%2F%2Fidenticons.github.com%2F0ec10d5e3ed3720a2d578417a894cf49.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="rxin" href="/apache/incubator-spark/commits/master/core/pom.xml?author=rxin"><img height="20" src="https://0.gravatar.com/avatar/73bfac91877fce35c6d20e16e9e53677?d=https%3A%2F%2Fidenticons.github.com%2F017ef876ba320362d11baf01bb584eee.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="tomdz" href="/apache/incubator-spark/commits/master/core/pom.xml?author=tomdz"><img height="20" src="https://2.gravatar.com/avatar/18039bff98071ad398b4301cfb0522b4?d=https%3A%2F%2Fidenticons.github.com%2F95b22f6d3c85b4945466aee72a712a01.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="shivaram" href="/apache/incubator-spark/commits/master/core/pom.xml?author=shivaram"><img height="20" src="https://0.gravatar.com/avatar/9d61244268a5be0ab361dc6e4af65ee4?d=https%3A%2F%2Fidenticons.github.com%2F020255c0f2993d01f44c91e0b508fa84.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="mridulm" href="/apache/incubator-spark/commits/master/core/pom.xml?author=mridulm"><img height="20" src="https://1.gravatar.com/avatar/0c6f3ad3eb45b1c42ca026f9d6ff3794?d=https%3A%2F%2Fidenticons.github.com%2Feed4d497b06cb847301ab49f40608dd4.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="woggle" href="/apache/incubator-spark/commits/master/core/pom.xml?author=woggle"><img height="20" src="https://1.gravatar.com/avatar/86f3647dccd886de735435991f55848e?d=https%3A%2F%2Fidenticons.github.com%2Ff23fdb10552dbc5ec1a5c9ba8e6f6be3.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="colorant" href="/apache/incubator-spark/commits/master/core/pom.xml?author=colorant"><img height="20" src="https://0.gravatar.com/avatar/1e54a90ff3671ca36be8a1163038ca56?d=https%3A%2F%2Fidenticons.github.com%2F90d602b3c6ec06eda12e485e109415fe.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="jerryshao" href="/apache/incubator-spark/commits/master/core/pom.xml?author=jerryshao"><img height="20" src="https://2.gravatar.com/avatar/24fb58dab7ef7c3202c4c8061ee51a12?d=https%3A%2F%2Fidenticons.github.com%2Ff8737f93e1f940e20f1259ad5403be32.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="russellcardullo" href="/apache/incubator-spark/commits/master/core/pom.xml?author=russellcardullo"><img height="20" src="https://0.gravatar.com/avatar/4173fab43d50a19e42950a8f7c7d96f8?d=https%3A%2F%2Fidenticons.github.com%2Fcae42ebaab97a0f8a1feb2f3fc97ee91.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="folone" href="/apache/incubator-spark/commits/master/core/pom.xml?author=folone"><img height="20" src="https://2.gravatar.com/avatar/50e7e3f60b3507383d2b327857b66a62?d=https%3A%2F%2Fidenticons.github.com%2F6c2bcbccd23191b40f4932e2b8450681.png&r=x&s=140" width="20" /></a> + <a class="avatar tooltipped downwards" title="witgo" href="/apache/incubator-spark/commits/master/core/pom.xml?author=witgo"><img height="20" src="https://2.gravatar.com/avatar/82f14c57ba7522241b0e3c67d759609e?d=https%3A%2F%2Fidenticons.github.com%2F878f3103ff85a3edb2d415930bbdbd5a.png&r=x&s=140" width="20" /></a> + + + </div> + <div id="blob_contributors_box" style="display:none"> + <h2 class="facebox-header">Users who have contributed to this file</h2> + <ul class="facebox-user-list"> + <li class="facebox-user-list-item"> + <img height="24" src="https://1.gravatar.com/avatar/e9813bbbab2caa993bf7e2b2d60de894?d=https%3A%2F%2Fidenticons.github.com%2F38c660c74f82a216b75167debab770ed.png&r=x&s=140" width="24" /> + <a href="/ScrapCodes">ScrapCodes</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://2.gravatar.com/avatar/17d3d634fc898bf3ae19444450af6805?d=https%3A%2F%2Fidenticons.github.com%2F627fcfc1b5e8e4535cc26ecfa133743e.png&r=x&s=140" width="24" /> + <a href="/mateiz">mateiz</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://1.gravatar.com/avatar/85a3f16c85ee6e38527e82e035ed4bf9?d=https%3A%2F%2Fidenticons.github.com%2F2ebb6c06bdc16ef37ec965c6b325b5c6.png&r=x&s=140" width="24" /> + <a href="/jey">jey</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://2.gravatar.com/avatar/aaa2e907159f6919c4a4c8039d46752f?d=https%3A%2F%2Fidenticons.github.com%2F80aa657180765c73a93528281452d8dc.png&r=x&s=140" width="24" /> + <a href="/markhamstra">markhamstra</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://2.gravatar.com/avatar/def597cee3897ba290da26972ff628d2?d=https%3A%2F%2Fidenticons.github.com%2F0ec10d5e3ed3720a2d578417a894cf49.png&r=x&s=140" width="24" /> + <a href="/pwendell">pwendell</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://0.gravatar.com/avatar/73bfac91877fce35c6d20e16e9e53677?d=https%3A%2F%2Fidenticons.github.com%2F017ef876ba320362d11baf01bb584eee.png&r=x&s=140" width="24" /> + <a href="/rxin">rxin</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://2.gravatar.com/avatar/18039bff98071ad398b4301cfb0522b4?d=https%3A%2F%2Fidenticons.github.com%2F95b22f6d3c85b4945466aee72a712a01.png&r=x&s=140" width="24" /> + <a href="/tomdz">tomdz</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://0.gravatar.com/avatar/9d61244268a5be0ab361dc6e4af65ee4?d=https%3A%2F%2Fidenticons.github.com%2F020255c0f2993d01f44c91e0b508fa84.png&r=x&s=140" width="24" /> + <a href="/shivaram">shivaram</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://1.gravatar.com/avatar/0c6f3ad3eb45b1c42ca026f9d6ff3794?d=https%3A%2F%2Fidenticons.github.com%2Feed4d497b06cb847301ab49f40608dd4.png&r=x&s=140" width="24" /> + <a href="/mridulm">mridulm</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://1.gravatar.com/avatar/86f3647dccd886de735435991f55848e?d=https%3A%2F%2Fidenticons.github.com%2Ff23fdb10552dbc5ec1a5c9ba8e6f6be3.png&r=x&s=140" width="24" /> + <a href="/woggle">woggle</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://0.gravatar.com/avatar/1e54a90ff3671ca36be8a1163038ca56?d=https%3A%2F%2Fidenticons.github.com%2F90d602b3c6ec06eda12e485e109415fe.png&r=x&s=140" width="24" /> + <a href="/colorant">colorant</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://2.gravatar.com/avatar/24fb58dab7ef7c3202c4c8061ee51a12?d=https%3A%2F%2Fidenticons.github.com%2Ff8737f93e1f940e20f1259ad5403be32.png&r=x&s=140" width="24" /> + <a href="/jerryshao">jerryshao</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://0.gravatar.com/avatar/4173fab43d50a19e42950a8f7c7d96f8?d=https%3A%2F%2Fidenticons.github.com%2Fcae42ebaab97a0f8a1feb2f3fc97ee91.png&r=x&s=140" width="24" /> + <a href="/russellcardullo">russellcardullo</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://2.gravatar.com/avatar/50e7e3f60b3507383d2b327857b66a62?d=https%3A%2F%2Fidenticons.github.com%2F6c2bcbccd23191b40f4932e2b8450681.png&r=x&s=140" width="24" /> + <a href="/folone">folone</a> + </li> + <li class="facebox-user-list-item"> + <img height="24" src="https://2.gravatar.com/avatar/82f14c57ba7522241b0e3c67d759609e?d=https%3A%2F%2Fidenticons.github.com%2F878f3103ff85a3edb2d415930bbdbd5a.png&r=x&s=140" width="24" /> + <a href="/witgo">witgo</a> + </li> + </ul> + </div> + </div> + +<div id="files" class="bubble"> + <div class="file"> + <div class="meta"> + <div class="info"> + <span class="icon"><b class="octicon octicon-file-text"></b></span> + <span class="mode" title="File Mode">file</span> + <span>232 lines (228 sloc)</span> + <span>7.715 kb</span> + </div> + <div class="actions"> + <div class="button-group"> + <a class="minibutton disabled tooltipped leftwards" href="#" + title="You must be signed in to make or propose changes">Edit</a> + <a href="/apache/incubator-spark/raw/master/core/pom.xml" class="button minibutton " id="raw-url">Raw</a> + <a href="/apache/incubator-spark/blame/master/core/pom.xml" class="button minibutton ">Blame</a> + <a href="/apache/incubator-spark/commits/master/core/pom.xml" class="button minibutton " rel="nofollow">History</a> + </div><!-- /.button-group --> + <a class="minibutton danger disabled empty-icon tooltipped leftwards" href="#" + title="You must be signed in to make or propose changes"> + Delete + </a> + </div><!-- /.actions --> + + </div> + <div class="blob-wrapper data type-xml js-blob-data"> + <table class="file-code file-diff"> + <tr class="file-code-line"> + <td class="blob-line-nums"> + <span id="L1" rel="#L1">1</span> +<span id="L2" rel="#L2">2</span> +<span id="L3" rel="#L3">3</span> +<span id="L4" rel="#L4">4</span> +<span id="L5" rel="#L5">5</span> +<span id="L6" rel="#L6">6</span> +<span id="L7" rel="#L7">7</span> +<span id="L8" rel="#L8">8</span> +<span id="L9" rel="#L9">9</span> +<span id="L10" rel="#L10">10</span> +<span id="L11" rel="#L11">11</span> +<span id="L12" rel="#L12">12</span> +<span id="L13" rel="#L13">13</span> +<span id="L14" rel="#L14">14</span> +<span id="L15" rel="#L15">15</span> +<span id="L16" rel="#L16">16</span> +<span id="L17" rel="#L17">17</span> +<span id="L18" rel="#L18">18</span> +<span id="L19" rel="#L19">19</span> +<span id="L20" rel="#L20">20</span> +<span id="L21" rel="#L21">21</span> +<span id="L22" rel="#L22">22</span> +<span id="L23" rel="#L23">23</span> +<span id="L24" rel="#L24">24</span> +<span id="L25" rel="#L25">25</span> +<span id="L26" rel="#L26">26</span> +<span id="L27" rel="#L27">27</span> +<span id="L28" rel="#L28">28</span> +<span id="L29" rel="#L29">29</span> +<span id="L30" rel="#L30">30</span> +<span id="L31" rel="#L31">31</span> +<span id="L32" rel="#L32">32</span> +<span id="L33" rel="#L33">33</span> +<span id="L34" rel="#L34">34</span> +<span id="L35" rel="#L35">35</span> +<span id="L36" rel="#L36">36</span> +<span id="L37" rel="#L37">37</span> +<span id="L38" rel="#L38">38</span> +<span id="L39" rel="#L39">39</span> +<span id="L40" rel="#L40">40</span> +<span id="L41" rel="#L41">41</span> +<span id="L42" rel="#L42">42</span> +<span id="L43" rel="#L43">43</span> +<span id="L44" rel="#L44">44</span> +<span id="L45" rel="#L45">45</span> +<span id="L46" rel="#L46">46</span> +<span id="L47" rel="#L47">47</span> +<span id="L48" rel="#L48">48</span> +<span id="L49" rel="#L49">49</span> +<span id="L50" rel="#L50">50</span> +<span id="L51" rel="#L51">51</span> +<span id="L52" rel="#L52">52</span> +<span id="L53" rel="#L53">53</span> +<span id="L54" rel="#L54">54</span> +<span id="L55" rel="#L55">55</span> +<span id="L56" rel="#L56">56</span> +<span id="L57" rel="#L57">57</span> +<span id="L58" rel="#L58">58</span> +<span id="L59" rel="#L59">59</span> +<span id="L60" rel="#L60">60</span> +<span id="L61" rel="#L61">61</span> +<span id="L62" rel="#L62">62</span> +<span id="L63" rel="#L63">63</span> +<span id="L64" rel="#L64">64</span> +<span id="L65" rel="#L65">65</span> +<span id="L66" rel="#L66">66</span> +<span id="L67" rel="#L67">67</span> +<span id="L68" rel="#L68">68</span> +<span id="L69" rel="#L69">69</span> +<span id="L70" rel="#L70">70</span> +<span id="L71" rel="#L71">71</span> +<span id="L72" rel="#L72">72</span> +<span id="L73" rel="#L73">73</span> +<span id="L74" rel="#L74">74</span> +<span id="L75" rel="#L75">75</span> +<span id="L76" rel="#L76">76</span> +<span id="L77" rel="#L77">77</span> +<span id="L78" rel="#L78">78</span> +<span id="L79" rel="#L79">79</span> +<span id="L80" rel="#L80">80</span> +<span id="L81" rel="#L81">81</span> +<span id="L82" rel="#L82">82</span> +<span id="L83" rel="#L83">83</span> +<span id="L84" rel="#L84">84</span> +<span id="L85" rel="#L85">85</span> +<span id="L86" rel="#L86">86</span> +<span id="L87" rel="#L87">87</span> +<span id="L88" rel="#L88">88</span> +<span id="L89" rel="#L89">89</span> +<span id="L90" rel="#L90">90</span> +<span id="L91" rel="#L91">91</span> +<span id="L92" rel="#L92">92</span> +<span id="L93" rel="#L93">93</span> +<span id="L94" rel="#L94">94</span> +<span id="L95" rel="#L95">95</span> +<span id="L96" rel="#L96">96</span> +<span id="L97" rel="#L97">97</span> +<span id="L98" rel="#L98">98</span> +<span id="L99" rel="#L99">99</span> +<span id="L100" rel="#L100">100</span> +<span id="L101" rel="#L101">101</span> +<span id="L102" rel="#L102">102</span> +<span id="L103" rel="#L103">103</span> +<span id="L104" rel="#L104">104</span> +<span id="L105" rel="#L105">105</span> +<span id="L106" rel="#L106">106</span> +<span id="L107" rel="#L107">107</span> +<span id="L108" rel="#L108">108</span> +<span id="L109" rel="#L109">109</span> +<span id="L110" rel="#L110">110</span> +<span id="L111" rel="#L111">111</span> +<span id="L112" rel="#L112">112</span> +<span id="L113" rel="#L113">113</span> +<span id="L114" rel="#L114">114</span> +<span id="L115" rel="#L115">115</span> +<span id="L116" rel="#L116">116</span> +<span id="L117" rel="#L117">117</span> +<span id="L118" rel="#L118">118</span> +<span id="L119" rel="#L119">119</span> +<span id="L120" rel="#L120">120</span> +<span id="L121" rel="#L121">121</span> +<span id="L122" rel="#L122">122</span> +<span id="L123" rel="#L123">123</span> +<span id="L124" rel="#L124">124</span> +<span id="L125" rel="#L125">125</span> +<span id="L126" rel="#L126">126</span> +<span id="L127" rel="#L127">127</span> +<span id="L128" rel="#L128">128</span> +<span id="L129" rel="#L129">129</span> +<span id="L130" rel="#L130">130</span> +<span id="L131" rel="#L131">131</span> +<span id="L132" rel="#L132">132</span> +<span id="L133" rel="#L133">133</span> +<span id="L134" rel="#L134">134</span> +<span id="L135" rel="#L135">135</span> +<span id="L136" rel="#L136">136</span> +<span id="L137" rel="#L137">137</span> +<span id="L138" rel="#L138">138</span> +<span id="L139" rel="#L139">139</span> +<span id="L140" rel="#L140">140</span> +<span id="L141" rel="#L141">141</span> +<span id="L142" rel="#L142">142</span> +<span id="L143" rel="#L143">143</span> +<span id="L144" rel="#L144">144</span> +<span id="L145" rel="#L145">145</span> +<span id="L146" rel="#L146">146</span> +<span id="L147" rel="#L147">147</span> +<span id="L148" rel="#L148">148</span> +<span id="L149" rel="#L149">149</span> +<span id="L150" rel="#L150">150</span> +<span id="L151" rel="#L151">151</span> +<span id="L152" rel="#L152">152</span> +<span id="L153" rel="#L153">153</span> +<span id="L154" rel="#L154">154</span> +<span id="L155" rel="#L155">155</span> +<span id="L156" rel="#L156">156</span> +<span id="L157" rel="#L157">157</span> +<span id="L158" rel="#L158">158</span> +<span id="L159" rel="#L159">159</span> +<span id="L160" rel="#L160">160</span> +<span id="L161" rel="#L161">161</span> +<span id="L162" rel="#L162">162</span> +<span id="L163" rel="#L163">163</span> +<span id="L164" rel="#L164">164</span> +<span id="L165" rel="#L165">165</span> +<span id="L166" rel="#L166">166</span> +<span id="L167" rel="#L167">167</span> +<span id="L168" rel="#L168">168</span> +<span id="L169" rel="#L169">169</span> +<span id="L170" rel="#L170">170</span> +<span id="L171" rel="#L171">171</span> +<span id="L172" rel="#L172">172</span> +<span id="L173" rel="#L173">173</span> +<span id="L174" rel="#L174">174</span> +<span id="L175" rel="#L175">175</span> +<span id="L176" rel="#L176">176</span> +<span id="L177" rel="#L177">177</span> +<span id="L178" rel="#L178">178</span> +<span id="L179" rel="#L179">179</span> +<span id="L180" rel="#L180">180</span> +<span id="L181" rel="#L181">181</span> +<span id="L182" rel="#L182">182</span> +<span id="L183" rel="#L183">183</span> +<span id="L184" rel="#L184">184</span> +<span id="L185" rel="#L185">185</span> +<span id="L186" rel="#L186">186</span> +<span id="L187" rel="#L187">187</span> +<span id="L188" rel="#L188">188</span> +<span id="L189" rel="#L189">189</span> +<span id="L190" rel="#L190">190</span> +<span id="L191" rel="#L191">191</span> +<span id="L192" rel="#L192">192</span> +<span id="L193" rel="#L193">193</span> +<span id="L194" rel="#L194">194</span> +<span id="L195" rel="#L195">195</span> +<span id="L196" rel="#L196">196</span> +<span id="L197" rel="#L197">197</span> +<span id="L198" rel="#L198">198</span> +<span id="L199" rel="#L199">199</span> +<span id="L200" rel="#L200">200</span> +<span id="L201" rel="#L201">201</span> +<span id="L202" rel="#L202">202</span> +<span id="L203" rel="#L203">203</span> +<span id="L204" rel="#L204">204</span> +<span id="L205" rel="#L205">205</span> +<span id="L206" rel="#L206">206</span> +<span id="L207" rel="#L207">207</span> +<span id="L208" rel="#L208">208</span> +<span id="L209" rel="#L209">209</span> +<span id="L210" rel="#L210">210</span> +<span id="L211" rel="#L211">211</span> +<span id="L212" rel="#L212">212</span> +<span id="L213" rel="#L213">213</span> +<span id="L214" rel="#L214">214</span> +<span id="L215" rel="#L215">215</span> +<span id="L216" rel="#L216">216</span> +<span id="L217" rel="#L217">217</span> +<span id="L218" rel="#L218">218</span> +<span id="L219" rel="#L219">219</span> +<span id="L220" rel="#L220">220</span> +<span id="L221" rel="#L221">221</span> +<span id="L222" rel="#L222">222</span> +<span id="L223" rel="#L223">223</span> +<span id="L224" rel="#L224">224</span> +<span id="L225" rel="#L225">225</span> +<span id="L226" rel="#L226">226</span> +<span id="L227" rel="#L227">227</span> +<span id="L228" rel="#L228">228</span> +<span id="L229" rel="#L229">229</span> +<span id="L230" rel="#L230">230</span> +<span id="L231" rel="#L231">231</span> + + </td> + <td class="blob-line-code"> + <div class="code-body highlight"><pre><div class='line' id='LC1'><span class="cp"><?xml version="1.0" encoding="UTF-8"?></span></div><div class='line' id='LC2'><span class="c"><!--</span></div><div class='line' id='LC3'><span class="c"> ~ Licensed to the Apache Software Foundation (ASF) under one or more</span></div><div class='line' id='LC4'><span class="c"> ~ contributor license agreements. See the NOTICE file distributed with</span></div><div class='line' id='LC5'><span class="c"> ~ this work for additional information regarding copyright ownership.</span></div><div class='line' id='LC6'><span class="c"> ~ The ASF licenses this file to You under the Apache License, Version 2.0</span></div><div class='line' id='LC7'><span class="c"> ~ (the "License"); you may not use this file except in compliance with</span></div><div class='line' id='LC8'><span class="c"> ~ the License. You may obtain a copy of the License at</span></div><div class='line' id='LC9'><span class="c"> ~</span></div><div class='line' id='LC10'><span class="c"> ~ http://www.apache.org/licenses/LICENSE-2.0</span></div><div class='line' id='LC11'><span class="c"> ~</span></div><div class='line' id='LC12'><span class="c"> ~ Unless required by applicable law or agreed to in writing, software</span></div><div class='line' id='LC13'><span class="c"> ~ distributed under the License is distributed on an "AS IS" BASIS,</span></div><div class='line' id='LC14'><span class="c"> ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span></div><div class='line' id='LC15'><span class="c"> ~ See the License for the specific language governing permissions and</span></div><div class='line' id='LC16'><span class="c"> ~ limitations under the License.</span></div><div class='line' id='LC17'><span class="c"> --></span></div><div class='line' id='LC18'><br/></div><div class='line' id='LC19'><span class="nt"><project</span> <span class="na">xmlns=</span><span class="s">"http://maven.apache.org/POM/4.0.0"</span> <span class="na">xmlns:xsi=</span><span class="s">"http://www.w3.org/2001/XMLSchema-instance"</span> <span class="na">xsi:schemaLocation=</span><span class="s">"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"</span><span class="nt">></span></div><div class='line' id='LC20'> <span class="nt"><modelVersion></span>4.0.0<span class="nt"></modelVersion></span></div><div class='line' id='LC21'> <span class="nt"><parent></span></div><div class='line' id='LC22'> <span class="nt"><groupId></span>org.apache.spark<span class="nt"></groupId></span></div><div class='line' id='LC23'> <span class="nt"><artifactId></span>spark-parent<span class="nt"></artifactId></span></div><div class='line' id='LC24'> <span class="nt"><version></span>0.9.0-incubating-SNAPSHOT<span class="nt"></version></span></div><div class='line' id='LC25'> <span class="nt"><relativePath></span>../pom.xml<span class="nt"></relativePath></span></div><div class='line' id='LC26'> <span class="nt"></parent></span></div><div class='line' id='LC27'><br/></div><div class='line' id='LC28'> <span class="nt"><groupId></span>org.apache.spark<span class="nt"></groupId></span></div><div class='line' id='LC29'> <span class="nt"><artifactId></span>spark-core_2.10<span class="nt"></artifactId></span></div><div class='line' id='LC30'> <span class="nt"><packaging></span>jar<span class="nt"></packaging></span></div><div class='line' id='LC31'> <span class="nt"><name></span>Spark Project Core<span class="nt"></name></span></div><div class='line' id='LC32'> <span class="nt"><url></span>http://spark.incubator.apache.org/<span class="nt"></url></span></div><div class='line' id='LC33'><br/></div><div class='line' id='LC34'> <span class="nt"><dependencies></span></div><div class='line' id='LC35'> <span class="nt"><dependency></span></div><div class='line' id='LC36'> <span class="nt"><groupId></span>org.apache.hadoop<span class="nt"></groupId></span></div><div class='line' id='LC37'> <span class="nt"><artifactId></span>hadoop-client<span class="nt"></artifactId></span></div><div class='line' id='LC38'> <span class="nt"></dependency></span></div><div class='line' id='LC39'> <span class="nt"><dependency></span></div><div class='line' id='LC40'> <span class="nt"><groupId></span>net.java.dev.jets3t<span class="nt"></groupId></span></div><div class='line' id='LC41'> <span class="nt"><artifactId></span>jets3t<span class="nt"></artifactId></span></div><div class='line' id='LC42'> <span class="nt"></dependency></span></div><div class='line' id='LC43'> <span class="nt"><dependency></span></div><div class='line' id='LC44'> <span class="nt"><groupId></span>org.apache.avro<span class="nt"></groupId></span></div><div class='line' id='LC45'> <span class="nt"><artifactId></span>avro<span class="nt"></artifactId></span></div><div class='line' id='LC46'> <span class="nt"></dependency></span></div><div class='line' id='LC47'> <span class="nt"><dependency></span></div><div class='line' id='LC48'> <span class="nt"><groupId></span>org.apache.avro<span class="nt"></groupId></span></div><div class='line' id='LC49'> <span class="nt"><artifactId></span>avro-ipc<span class="nt"></artifactId></span></div><div class='line' id='LC50'> <span class="nt"></dependency></span></div><div class='line' id='LC51'> <span class="nt"><dependency></span></div><div class='line' id='LC52'> <span class="nt"><groupId></span>org.apache.zookeeper<span class="nt"></groupId></span></div><div class='line' id='LC53'> <span class="nt"><artifactId></span>zookeeper<span class="nt"></artifactId></span></div><div class='line' id='LC54'> <span class="nt"></dependency></span></div><div class='line' id='LC55'> <span class="nt"><dependency></span></div><div class='line' id='LC56'> <span class="nt"><groupId></span>org.eclipse.jetty<span class="nt"></groupId></span></div><div class='line' id='LC57'> <span class="nt"><artifactId></span>jetty-server<span class="nt"></artifactId></span></div><div class='line' id='LC58'> <span class="nt"></dependency></span></div><div class='line' id='LC59'> <span class="nt"><dependency></span></div><div class='line' id='LC60'> <span class="nt"><groupId></span>com.google.guava<span class="nt"></groupId></span></div><div class='line' id='LC61'> <span class="nt"><artifactId></span>guava<span class="nt"></artifactId></span></div><div class='line' id='LC62'> <span class="nt"></dependency></span></div><div class='line' id='LC63'> <span class="nt"><dependency></span></div><div class='line' id='LC64'> <span class="nt"><groupId></span>com.google.code.findbugs<span class="nt"></groupId></span></div><div class='line' id='LC65'> <span class="nt"><artifactId></span>jsr305<span class="nt"></artifactId></span></div><div class='line' id='LC66'> <span class="nt"></dependency></span></div><div class='line' id='LC67'> <span class="nt"><dependency></span></div><div class='line' id='LC68'> <span class="nt"><groupId></span>org.slf4j<span class="nt"></groupId></span></div><div class='line' id='LC69'> <span class="nt"><artifactId></span>slf4j-api<span class="nt"></artifactId></span></div><div class='line' id='LC70'> <span class="nt"></dependency></span></div><div class='line' id='LC71'> <span class="nt"><dependency></span></div><div class='line' id='LC72'> <span class="nt"><groupId></span>com.ning<span class="nt"></groupId></span></div><div class='line' id='LC73'> <span class="nt"><artifactId></span>compress-lzf<span class="nt"></artifactId></span></div><div class='line' id='LC74'> <span class="nt"></dependency></span></div><div class='line' id='LC75'> <span class="nt"><dependency></span></div><div class='line' id='LC76'> <span class="nt"><groupId></span>org.xerial.snappy<span class="nt"></groupId></span></div><div class='line' id='LC77'> <span class="nt"><artifactId></span>snappy-java<span class="nt"></artifactId></span></div><div class='line' id='LC78'> <span class="nt"></dependency></span></div><div class='line' id='LC79'> <span class="nt"><dependency></span></div><div class='line' id='LC80'> <span class="nt"><groupId></span>org.ow2.asm<span class="nt"></groupId></span></div><div class='line' id='LC81'> <span class="nt"><artifactId></span>asm<span class="nt"></artifactId></span></div><div class='line' id='LC82'> <span class="nt"></dependency></span></div><div class='line' id='LC83'> <span class="nt"><dependency></span></div><div class='line' id='LC84'> <span class="nt"><groupId></span>com.twitter<span class="nt"></groupId></span></div><div class='line' id='LC85'> <span class="nt"><artifactId></span>chill_${scala.binary.version}<span class="nt"></artifactId></span></div><div class='line' id='LC86'> <span class="nt"><version></span>0.3.1<span class="nt"></version></span></div><div class='line' id='LC87'> <span class="nt"></dependency></span></div><div class='line' id='LC88'> <span class="nt"><dependency></span></div><div class='line' id='LC89'> <span class="nt"><groupId></span>com.twitter<span class="nt"></groupId></span></div><div class='line' id='LC90'> <span class="nt"><artifactId></span>chill-java<span class="nt"></artifactId></span></div><div class='line' id='LC91'> <span class="nt"><version></span>0.3.1<span class="nt"></version></span></div><div class='line' id='LC92'> <span class="nt"></dependency></span></div><div class='line' id='LC93'> <span class="nt"><dependency></span></div><div class='line' id='LC94'> <span class="nt"><groupId></span>${akka.group}<span class="nt"></groupId></span></div><div class='line' id='LC95'> <span class="nt"><artifactId></span>akka-remote_${scala.binary.version}<span class="nt"></artifactId></span></div><div class='line' id='LC96'> <span class="nt"></dependency></span></div><div class='line' id='LC97'> <span class="nt"><dependency></span></div><div class='line' id='LC98'> <span class="nt"><groupId></span>${akka.group}<span class="nt"></groupId></span></div><div class='line' id='LC99'> <span class="nt"><artifactId></span>akka-slf4j_${scala.binary.version}<span class="nt"></artifactId></span></div><div class='line' id='LC100'> <span class="nt"></dependency></span></div><div class='line' id='LC101'> <span class="nt"><dependency></span></div><div class='line' id='LC102'> <span class="nt"><groupId></span>org.scala-lang<span class="nt"></groupId></span></div><div class='line' id='LC103'> <span class="nt"><artifactId></span>scala-library<span class="nt"></artifactId></span></div><div class='line' id='LC104'> <span class="nt"></dependency></span></div><div class='line' id='LC105'> <span class="nt"><dependency></span></div><div class='line' id='LC106'> <span class="nt"><groupId></span>net.liftweb<span class="nt"></groupId></span></div><div class='line' id='LC107'> <span class="nt"><artifactId></span>lift-json_${scala.binary.version}<span class="nt"></artifactId></span></div><div class='line' id='LC108'> <span class="nt"></dependency></span></div><div class='line' id='LC109'> <span class="nt"><dependency></span></div><div class='line' id='LC110'> <span class="nt"><groupId></span>it.unimi.dsi<span class="nt"></groupId></span></div><div class='line' id='LC111'> <span class="nt"><artifactId></span>fastutil<span class="nt"></artifactId></span></div><div class='line' id='LC112'> <span class="nt"></dependency></span></div><div class='line' id='LC113'> <span class="nt"><dependency></span></div><div class='line' id='LC114'> <span class="nt"><groupId></span>colt<span class="nt"></groupId></span></div><div class='line' id='LC115'> <span class="nt"><artifactId></span>colt<span class="nt"></artifactId></span></div><div class='line' id='LC116'> <span class="nt"></dependency></span></div><div class='line' id='LC117'> <span class="nt"><dependency></span></div><div class='line' id='LC118'> <span class="nt"><groupId></span>org.apache.mesos<span class="nt"></groupId></span></div><div class='line' id='LC119'> <span class="nt"><artifactId></span>mesos<span class="nt"></artifactId></span></div><div class='line' id='LC120'> <span class="nt"></dependency></span></div><div class='line' id='LC121'> <span class="nt"><dependency></span></div><div class='line' id='LC122'> <span class="nt"><groupId></span>io.netty<span class="nt"></groupId></span></div><div class='line' id='LC123'> <span class="nt"><artifactId></span>netty-all<span class="nt"></artifactId></span></div><div class='line' id='LC124'> <span class="nt"></dependency></span></div><div class='line' id='LC125'> <span class="nt"><dependency></span></div><div class='line' id='LC126'> <span class="nt"><groupId></span>log4j<span class="nt"></groupId></span></div><div class='line' id='LC127'> <span class="nt"><artifactId></span>log4j<span class="nt"></artifactId></span></div><div class='line' id='LC128'> <span class="nt"></dependency></span></div><div class='line' id='LC129'> <span class="nt"><dependency></span></div><div class='line' id='LC130'> <span class="nt"><groupId></span>com.codahale.metrics<span class="nt"></groupId></span></div><div class='line' id='LC131'> <span class="nt"><artifactId></span>metrics-core<span class="nt"></artifactId></span></div><div class='line' id='LC132'> <span class="nt"></dependency></span></div><div class='line' id='LC133'> <span class="nt"><dependency></span></div><div class='line' id='LC134'> <span class="nt"><groupId></span>com.codahale.metrics<span class="nt"></groupId></span></div><div class='line' id='LC135'> <span class="nt"><artifactId></span>metrics-jvm<span class="nt"></artifactId></span></div><div class='line' id='LC136'> <span class="nt"></dependency></span></div><div class='line' id='LC137'> <span class="nt"><dependency></span></div><div class='line' id='LC138'> <span class="nt"><groupId></span>com.codahale.metrics<span class="nt"></groupId></span></div><div class='line' id='LC139'> <span class="nt"><artifactId></span>metrics-json<span class="nt"></artifactId></span></div><div class='line' id='LC140'> <span class="nt"></dependency></span></div><div class='line' id='LC141'> <span class="nt"><dependency></span></div><div class='line' id='LC142'> <span class="nt"><groupId></span>com.codahale.metrics<span class="nt"></groupId></span></div><div class='line' id='LC143'> <span class="nt"><artifactId></span>metrics-ganglia<span class="nt"></artifactId></span></div><div class='line' id='LC144'> <span class="nt"></dependency></span></div><div class='line' id='LC145'> <span class="nt"><dependency></span></div><div class='line' id='LC146'> <span class="nt"><groupId></span>com.codahale.metrics<span class="nt"></groupId></span></div><div class='line' id='LC147'> <span class="nt"><artifactId></span>metrics-graphite<span class="nt"></artifactId></span></div><div class='line' id='LC148'> <span class="nt"></dependency></span></div><div class='line' id='LC149'> <span class="nt"><dependency></span></div><div class='line' id='LC150'> <span class="nt"><groupId></span>org.apache.derby<span class="nt"></groupId></span></div><div class='line' id='LC151'> <span class="nt"><artifactId></span>derby<span class="nt"></artifactId></span></div><div class='line' id='LC152'> <span class="nt"><scope></span>test<span class="nt"></scope></span></div><div class='line' id='LC153'> <span class="nt"></dependency></span></div><div class='line' id='LC154'> <span class="nt"><dependency></span></div><div class='line' id='LC155'> <span class="nt"><groupId></span>commons-io<span class="nt"></groupId></span></div><div class='line' id='LC156'> <span class="nt"><artifactId></span>commons-io<span class="nt"></artifactId></span></div><div class='line' id='LC157'> <span class="nt"><scope></span>test<span class="nt"></scope></span></div><div class='line' id='LC158'> <span class="nt"></dependency></span></div><div class='line' id='LC159'> <span class="nt"><dependency></span></div><div class='line' id='LC160'> <span class="nt"><groupId></span>org.scalatest<span class="nt"></groupId></span></div><div class='line' id='LC161'> <span class="nt"><artifactId></span>scalatest_${scala.binary.version}<span class="nt"></artifactId></span></div><div class='line' id='LC162'> <span class="nt"><scope></span>test<span class="nt"></scope></span></div><div class='line' id='LC163'> <span class="nt"></dependency></span></div><div class='line' id='LC164'> <span class="nt"><dependency></span></div><div class='line' id='LC165'> <span class="nt"><groupId></span>org.scalacheck<span class="nt"></groupId></span></div><div class='line' id='LC166'> <span class="nt"><artifactId></span>scalacheck_${scala.binary.version}<span class="nt"></artifactId></span></div><div class='line' id='LC167'> <span class="nt"><scope></span>test<span class="nt"></scope></span></div><div class='line' id='LC168'> <span class="nt"></dependency></span></div><div class='line' id='LC169'> <span class="nt"><dependency></span></div><div class='line' id='LC170'> <span class="nt"><groupId></span>org.easymock<span class="nt"></groupId></span></div><div class='line' id='LC171'> <span class="nt"><artifactId></span>easymock<span class="nt"></artifactId></span></div><div class='line' id='LC172'> <span class="nt"><scope></span>test<span class="nt"></scope></span></div><div class='line' id='LC173'> <span class="nt"></dependency></span></div><div class='line' id='LC174'> <span class="nt"><dependency></span></div><div class='line' id='LC175'> <span class="nt"><groupId></span>com.novocode<span class="nt"></groupId></span></div><div class='line' id='LC176'> <span class="nt"><artifactId></span>junit-interface<span class="nt"></artifactId></span></div><div class='line' id='LC177'> <span class="nt"><scope></span>test<span class="nt"></scope></span></div><div class='line' id='LC178'> <span class="nt"></dependency></span></div><div class='line' id='LC179'> <span class="nt"><dependency></span></div><div class='line' id='LC180'> <span class="nt"><groupId></span>org.slf4j<span class="nt"></groupId></span></div><div class='line' id='LC181'> <span class="nt"><artifactId></span>slf4j-log4j12<span class="nt"></artifactId></span></div><div class='line' id='LC182'> <span class="nt"><scope></span>test<span class="nt"></scope></span></div><div class='line' id='LC183'> <span class="nt"></dependency></span></div><div class='line' id='LC184'> <span class="nt"></dependencies></span></div><div class='line' id='LC185'> <span class="nt"><build></span></div><div class='line' id='LC186'> <span class="nt"><outputDirectory></span>target/scala-${scala.binary.version}/classes<span class="nt"></outputDirectory></span></div><div class='line' id='LC187'> <span class="nt"><testOutputDirectory></span>target/scala-${scala.binary.version}/test-classes<span class="nt"></testOutputDirectory></span></div><div class='line' id='LC188'> <span class="nt"><plugins></span></div><div class='line' id='LC189'> <span class="nt"><plugin></span></div><div class='line' id='LC190'> <span class="nt"><groupId></span>org.apache.maven.plugins<span class="nt"></groupId></span></div><div class='line' id='LC191'> <span class="nt"><artifactId></span>maven-antrun-plugin<span class="nt"></artifactId></span></div><div class='line' id='LC192'> <span class="nt"><executions></span></div><div class='line' id='LC193'> <span class="nt"><execution></span></div><div class='line' id='LC194'> <span class="nt"><phase></span>test<span class="nt"></phase></span></div><div class='line' id='LC195'> <span class="nt"><goals></span></div><div class='line' id='LC196'> <span class="nt"><goal></span>run<span class="nt"></goal></span></div><div class='line' id='LC197'> <span class="nt"></goals></span></div><div class='line' id='LC198'> <span class="nt"><configuration></span></div><div class='line' id='LC199'> <span class="nt"><exportAntProperties></span>true<span class="nt"></exportAntProperties></span></div><div class='line' id='LC200'> <span class="nt"><tasks></span></div><div class='line' id='LC201'> <span class="nt"><property</span> <span class="na">name=</span><span class="s">"spark.classpath"</span> <span class="na">refid=</span><span class="s">"maven.test.classpath"</span> <span class="nt">/></span></div><div class='line' id='LC202'> <span class="nt"><property</span> <span class="na">environment=</span><span class="s">"env"</span> <span class="nt">/></span></div><div class='line' id='LC203'> <span class="nt"><fail</span> <span class="na">message=</span><span class="s">"Please set the SCALA_HOME (or SCALA_LIBRARY_PATH if scala is on the path) environment variables and retry."</span><span class="nt">></span></div><div class='line' id='LC204'> <span class="nt"><condition></span></div><div class='line' id='LC205'> <span class="nt"><not></span></div><div class='line' id='LC206'> <span class="nt"><or></span></div><div class='line' id='LC207'> <span class="nt"><isset</span> <span class="na">property=</span><span class="s">"env.SCALA_HOME"</span> <span class="nt">/></span></div><div class='line' id='LC208'> <span class="nt"><isset</span> <span class="na">property=</span><span class="s">"env.SCALA_LIBRARY_PATH"</span> <span class="nt">/></span></div><div class='line' id='LC209'> <span class="nt"></or></span></div><div class='line' id='LC210'> <span class="nt"></not></span></div><div class='line' id='LC211'> <span class="nt"></condition></span></div><div class='line' id='LC212'> <span class="nt"></fail></span></div><div class='line' id='LC213'> <span class="nt"></tasks></span></div><div class='line' id='LC214'> <span class="nt"></configuration></span></div><div class='line' id='LC215'> <span class="nt"></execution></span></div><div class='line' id='LC216'> <span class="nt"></executions></span></div><div class='line' id='LC217'> <span class="nt"></plugin></span></div><div class='line' id='LC218'> <span class="nt"><plugin></span></div><div class='line' id='LC219'> <span class="nt"><groupId></span>org.scalatest<span class="nt"></groupId></span></div><div class='line' id='LC220'> <span class="nt"><artifactId></span>scalatest-maven-plugin<span class="nt"></artifactId></span></div><div class='line' id='LC221'> <span class="nt"><configuration></span></div><div class='line' id='LC222'> <span class="nt"><environmentVariables></span></div><div class='line' id='LC223'> <span class="nt"><SPARK_HOME></span>${basedir}/..<span class="nt"></SPARK_HOME></span></div><div class='line' id='LC224'> <span class="nt"><SPARK_TESTING></span>1<span class="nt"></SPARK_TESTING></span></div><div class='line' id='LC225'> <span class="nt"><SPARK_CLASSPATH></span>${spark.classpath}<span class="nt"></SPARK_CLASSPATH></span></div><div class='line' id='LC226'> <span class="nt"></environmentVariables></span></div><div class='line' id='LC227'> <span class="nt"></configuration></span></div><div class='line' id='LC228'> <span class="nt"></plugin></span></div><div class='line' id='LC229'> <span class="nt"></plugins></span></div><div class='line' id='LC230'> <span class="nt"></build></span></div><div class='line' id='LC231'><span class="nt"></project></span></div></pre></div> + </td> + </tr> + </table> + </div> + + </div> +</div> + +<a href="#jump-to-line" rel="facebox[.linejump]" data-hotkey="l" class="js-jump-to-line" style="display:none">Jump to Line</a> +<div id="jump-to-line" style="display:none"> + <form accept-charset="UTF-8" class="js-jump-to-line-form"> + <input class="linejump-input js-jump-to-line-field" type="text" placeholder="Jump to line…" autofocus> + <button type="submit" class="button">Go</button> + </form> +</div> + + </div> + + </div><!-- /.repo-container --> + <div class="modal-backdrop"></div> + </div><!-- /.container --> + </div><!-- /.site --> + + + </div><!-- /.wrapper --> + + <div class="container"> + <div class="site-footer"> + <ul class="site-footer-links right"> + <li><a href="https://status.github.com/">Status</a></li> + <li><a href="http://developer.github.com">API</a></li> + <li><a href="http://training.github.com">Training</a></li> + <li><a href="http://shop.github.com">Shop</a></li> + <li><a href="/blog">Blog</a></li> + <li><a href="/about">About</a></li> + + </ul> + + <a href="/"> + <span class="mega-octicon octicon-mark-github"></span> + </a> + + <ul class="site-footer-links"> + <li>© 2013 <span title="0.02474s from github-fe126-cp1-prd.iad.github.net">GitHub</span>, Inc.</li> + <li><a href="/site/terms">Terms</a></li> + <li><a href="/site/privacy">Privacy</a></li> + <li><a href="/security">Security</a></li> + <li><a href="/contact">Contact</a></li> + </ul> + </div><!-- /.site-footer --> +</div><!-- /.container --> + + + <div class="fullscreen-overlay js-fullscreen-overlay" id="fullscreen_overlay"> + <div class="fullscreen-container js-fullscreen-container"> + <div class="textarea-wrap"> + <textarea name="fullscreen-contents" id="fullscreen-contents" class="js-fullscreen-contents" placeholder="" data-suggester="fullscreen_suggester"></textarea> + <div class="suggester-container"> + <div class="suggester fullscreen-suggester js-navigation-container" id="fullscreen_suggester" + data-url="/apache/incubator-spark/suggestions/commit"> + </div> + </div> + </div> + </div> + <div class="fullscreen-sidebar"> + <a href="#" class="exit-fullscreen js-exit-fullscreen tooltipped leftwards" title="Exit Zen Mode"> + <span class="mega-octicon octicon-screen-normal"></span> + </a> + <a href="#" class="theme-switcher js-theme-switcher tooltipped leftwards" + title="Switch themes"> + <span class="octicon octicon-color-mode"></span> + </a> + </div> +</div> + + + + <div id="ajax-error-message" class="flash flash-error"> + <span class="octicon octicon-alert"></span> + <a href="#" class="octicon octicon-remove-close close ajax-error-dismiss"></a> + Something went wrong with that request. Please try again. + </div> + + </body> +</html> + diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala index 363667fa86..55c87450ac 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala @@ -611,6 +611,42 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])(implicit val kClassTag: ClassTag[K * Return an RDD with the values of each tuple. */ def values(): JavaRDD[V] = JavaRDD.fromRDD[V](rdd.map(_._2)) + + /** + * Return approximate number of distinct values for each key in this RDD. + * The accuracy of approximation can be controlled through the relative standard deviation + * (relativeSD) parameter, which also controls the amount of memory used. Lower values result in + * more accurate counts but increase the memory footprint and vise versa. Uses the provided + * Partitioner to partition the output RDD. + */ + def countApproxDistinctByKey(relativeSD: Double, partitioner: Partitioner): JavaRDD[(K, Long)] = { + rdd.countApproxDistinctByKey(relativeSD, partitioner) + } + + /** + * Return approximate number of distinct values for each key this RDD. + * The accuracy of approximation can be controlled through the relative standard deviation + * (relativeSD) parameter, which also controls the amount of memory used. Lower values result in + * more accurate counts but increase the memory footprint and vise versa. The default value of + * relativeSD is 0.05. Hash-partitions the output RDD using the existing partitioner/parallelism + * level. + */ + def countApproxDistinctByKey(relativeSD: Double = 0.05): JavaRDD[(K, Long)] = { + rdd.countApproxDistinctByKey(relativeSD) + } + + + /** + * Return approximate number of distinct values for each key in this RDD. + * The accuracy of approximation can be controlled through the relative standard deviation + * (relativeSD) parameter, which also controls the amount of memory used. Lower values result in + * more accurate counts but increase the memory footprint and vise versa. HashPartitions the + * output RDD into numPartitions. + * + */ + def countApproxDistinctByKey(relativeSD: Double, numPartitions: Int): JavaRDD[(K, Long)] = { + rdd.countApproxDistinctByKey(relativeSD, numPartitions) + } } object JavaPairRDD { diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala index f344804b4c..924d8af060 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala @@ -444,4 +444,15 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable { val comp = com.google.common.collect.Ordering.natural().asInstanceOf[Comparator[T]] takeOrdered(num, comp) } + + /** + * Return approximate number of distinct elements in the RDD. + * + * The accuracy of approximation can be controlled through the relative standard deviation + * (relativeSD) parameter, which also controls the amount of memory used. Lower values result in + * more accurate counts but increase the memory footprint and vise versa. The default value of + * relativeSD is 0.05. + */ + def countApproxDistinct(relativeSD: Double = 0.05): Long = rdd.countApproxDistinct(relativeSD) + } diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala index 48168e152e..04a8d05988 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala @@ -40,12 +40,15 @@ import org.apache.hadoop.mapreduce.SparkHadoopMapReduceUtil import org.apache.hadoop.mapreduce.{Job => NewAPIHadoopJob} import org.apache.hadoop.mapreduce.{RecordWriter => NewRecordWriter} +import com.clearspring.analytics.stream.cardinality.HyperLogLog + import org.apache.spark._ import org.apache.spark.SparkContext._ import org.apache.spark.partial.{BoundedDouble, PartialResult} import org.apache.spark.Aggregator import org.apache.spark.Partitioner import org.apache.spark.Partitioner.defaultPartitioner +import org.apache.spark.util.SerializableHyperLogLog /** * Extra functions available on RDDs of (key, value) pairs through an implicit conversion. @@ -208,6 +211,45 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) } /** + * Return approximate number of distinct values for each key in this RDD. + * The accuracy of approximation can be controlled through the relative standard deviation + * (relativeSD) parameter, which also controls the amount of memory used. Lower values result in + * more accurate counts but increase the memory footprint and vise versa. Uses the provided + * Partitioner to partition the output RDD. + */ + def countApproxDistinctByKey(relativeSD: Double, partitioner: Partitioner): RDD[(K, Long)] = { + val createHLL = (v: V) => new SerializableHyperLogLog(new HyperLogLog(relativeSD)).add(v) + val mergeValueHLL = (hll: SerializableHyperLogLog, v: V) => hll.add(v) + val mergeHLL = (h1: SerializableHyperLogLog, h2: SerializableHyperLogLog) => h1.merge(h2) + + combineByKey(createHLL, mergeValueHLL, mergeHLL, partitioner).mapValues(_.value.cardinality()) + } + + /** + * Return approximate number of distinct values for each key in this RDD. + * The accuracy of approximation can be controlled through the relative standard deviation + * (relativeSD) parameter, which also controls the amount of memory used. Lower values result in + * more accurate counts but increase the memory footprint and vise versa. HashPartitions the + * output RDD into numPartitions. + * + */ + def countApproxDistinctByKey(relativeSD: Double, numPartitions: Int): RDD[(K, Long)] = { + countApproxDistinctByKey(relativeSD, new HashPartitioner(numPartitions)) + } + + /** + * Return approximate number of distinct values for each key this RDD. + * The accuracy of approximation can be controlled through the relative standard deviation + * (relativeSD) parameter, which also controls the amount of memory used. Lower values result in + * more accurate counts but increase the memory footprint and vise versa. The default value of + * relativeSD is 0.05. Hash-partitions the output RDD using the existing partitioner/parallelism + * level. + */ + def countApproxDistinctByKey(relativeSD: Double = 0.05): RDD[(K, Long)] = { + countApproxDistinctByKey(relativeSD, defaultPartitioner(self)) + } + + /** * Merge the values for each key using an associative reduce function. This will also perform * the merging locally on each mapper before sending results to a reducer, similarly to a * "combiner" in MapReduce. Output will be hash-partitioned with numPartitions partitions. diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala index ea45566ad1..4960e6e82f 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala @@ -33,6 +33,7 @@ import org.apache.hadoop.io.Text import org.apache.hadoop.mapred.TextOutputFormat import it.unimi.dsi.fastutil.objects.{Object2LongOpenHashMap => OLMap} +import com.clearspring.analytics.stream.cardinality.HyperLogLog import org.apache.spark.Partitioner._ import org.apache.spark.api.java.JavaRDD @@ -41,7 +42,7 @@ import org.apache.spark.partial.CountEvaluator import org.apache.spark.partial.GroupedCountEvaluator import org.apache.spark.partial.PartialResult import org.apache.spark.storage.StorageLevel -import org.apache.spark.util.{Utils, BoundedPriorityQueue} +import org.apache.spark.util.{Utils, BoundedPriorityQueue, SerializableHyperLogLog} import org.apache.spark.SparkContext._ import org.apache.spark._ @@ -789,6 +790,19 @@ abstract class RDD[T: ClassTag]( } /** + * Return approximate number of distinct elements in the RDD. + * + * The accuracy of approximation can be controlled through the relative standard deviation + * (relativeSD) parameter, which also controls the amount of memory used. Lower values result in + * more accurate counts but increase the memory footprint and vise versa. The default value of + * relativeSD is 0.05. + */ + def countApproxDistinct(relativeSD: Double = 0.05): Long = { + val zeroCounter = new SerializableHyperLogLog(new HyperLogLog(relativeSD)) + aggregate(zeroCounter)(_.add(_), _.merge(_)).value.cardinality() + } + + /** * Take the first num elements of the RDD. It works by first scanning one partition, and use the * results from that partition to estimate the number of additional partitions needed to satisfy * the limit. diff --git a/core/src/main/scala/org/apache/spark/util/SerializableHyperLogLog.scala b/core/src/main/scala/org/apache/spark/util/SerializableHyperLogLog.scala new file mode 100644 index 0000000000..8b4e7c104c --- /dev/null +++ b/core/src/main/scala/org/apache/spark/util/SerializableHyperLogLog.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.util + +import java.io.{Externalizable, ObjectOutput, ObjectInput} +import com.clearspring.analytics.stream.cardinality.{ICardinality, HyperLogLog} + +/** + * A wrapper around [[com.clearspring.analytics.stream.cardinality.HyperLogLog]] that is serializable. + */ +private[spark] +class SerializableHyperLogLog(var value: ICardinality) extends Externalizable { + + def this() = this(null) // For deserialization + + def merge(other: SerializableHyperLogLog) = new SerializableHyperLogLog(value.merge(other.value)) + + def add[T](elem: T) = { + this.value.offer(elem) + this + } + + def readExternal(in: ObjectInput) { + val byteLength = in.readInt() + val bytes = new Array[Byte](byteLength) + in.readFully(bytes) + value = HyperLogLog.Builder.build(bytes) + } + + def writeExternal(out: ObjectOutput) { + val bytes = value.getBytes() + out.writeInt(bytes.length) + out.write(bytes) + } +} diff --git a/core/src/test/scala/org/apache/spark/JavaAPISuite.java b/core/src/test/scala/org/apache/spark/JavaAPISuite.java index 5e2899c97b..23ec6c3b31 100644 --- a/core/src/test/scala/org/apache/spark/JavaAPISuite.java +++ b/core/src/test/scala/org/apache/spark/JavaAPISuite.java @@ -930,4 +930,36 @@ public class JavaAPISuite implements Serializable { parts[1]); } + @Test + public void countApproxDistinct() { + List<Integer> arrayData = new ArrayList<Integer>(); + int size = 100; + for (int i = 0; i < 100000; i++) { + arrayData.add(i % size); + } + JavaRDD<Integer> simpleRdd = sc.parallelize(arrayData, 10); + Assert.assertTrue(Math.abs((simpleRdd.countApproxDistinct(0.2) - size) / (size * 1.0)) < 0.2); + Assert.assertTrue(Math.abs((simpleRdd.countApproxDistinct(0.05) - size) / (size * 1.0)) <= 0.05); + Assert.assertTrue(Math.abs((simpleRdd.countApproxDistinct(0.01) - size) / (size * 1.0)) <= 0.01); + } + + @Test + public void countApproxDistinctByKey() { + double relativeSD = 0.001; + + List<Tuple2<Integer, Integer>> arrayData = new ArrayList<Tuple2<Integer, Integer>>(); + for (int i = 10; i < 100; i++) + for (int j = 0; j < i; j++) + arrayData.add(new Tuple2<Integer, Integer>(i, j)); + + JavaPairRDD<Integer, Integer> pairRdd = sc.parallelizePairs(arrayData); + List<Tuple2<Integer, Object>> res = pairRdd.countApproxDistinctByKey(relativeSD).collect(); + for (Tuple2<Integer, Object> resItem : res) { + double count = (double)resItem._1(); + Long resCount = (Long)resItem._2(); + Double error = Math.abs((resCount - count) / count); + Assert.assertTrue(error < relativeSD); + } + + } } diff --git a/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala b/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala index 57d3382ed0..5da538a1dd 100644 --- a/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala +++ b/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.rdd import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.HashSet +import scala.util.Random import org.scalatest.FunSuite @@ -109,6 +110,39 @@ class PairRDDFunctionsSuite extends FunSuite with SharedSparkContext { assert(deps.size === 2) // ShuffledRDD, ParallelCollection. } + test("countApproxDistinctByKey") { + def error(est: Long, size: Long) = math.abs(est - size) / size.toDouble + + /* Since HyperLogLog unique counting is approximate, and the relative standard deviation is + * only a statistical bound, the tests can fail for large values of relativeSD. We will be using + * relatively tight error bounds to check correctness of functionality rather than checking + * whether the approximation conforms with the requested bound. + */ + val relativeSD = 0.001 + + // For each value i, there are i tuples with first element equal to i. + // Therefore, the expected count for key i would be i. + val stacked = (1 to 100).flatMap(i => (1 to i).map(j => (i, j))) + val rdd1 = sc.parallelize(stacked) + val counted1 = rdd1.countApproxDistinctByKey(relativeSD).collect() + counted1.foreach{ + case(k, count) => assert(error(count, k) < relativeSD) + } + + val rnd = new Random() + + // The expected count for key num would be num + val randStacked = (1 to 100).flatMap { i => + val num = rnd.nextInt % 500 + (1 to num).map(j => (num, j)) + } + val rdd2 = sc.parallelize(randStacked) + val counted2 = rdd2.countApproxDistinctByKey(relativeSD, 4).collect() + counted2.foreach{ + case(k, count) => assert(error(count, k) < relativeSD) + } + } + test("join") { val rdd1 = sc.parallelize(Array((1, 1), (1, 2), (2, 1), (3, 1))) val rdd2 = sc.parallelize(Array((1, 'x'), (2, 'y'), (2, 'z'), (4, 'w'))) diff --git a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala b/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala index d8dcd6d14c..1383359f85 100644 --- a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala +++ b/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala @@ -63,6 +63,19 @@ class RDDSuite extends FunSuite with SharedSparkContext { } } + test("countApproxDistinct") { + + def error(est: Long, size: Long) = math.abs(est - size) / size.toDouble + + val size = 100 + val uniformDistro = for (i <- 1 to 100000) yield i % size + val simpleRdd = sc.makeRDD(uniformDistro) + assert(error(simpleRdd.countApproxDistinct(0.2), size) < 0.2) + assert(error(simpleRdd.countApproxDistinct(0.05), size) < 0.05) + assert(error(simpleRdd.countApproxDistinct(0.01), size) < 0.01) + assert(error(simpleRdd.countApproxDistinct(0.001), size) < 0.001) + } + test("SparkContext.union") { val nums = sc.makeRDD(Array(1, 2, 3, 4), 2) assert(sc.union(nums).collect().toList === List(1, 2, 3, 4)) diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala index c016c51171..636e3ab913 100644 --- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala +++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala @@ -172,6 +172,10 @@ class KryoSerializerSuite extends FunSuite with SharedSparkContext { assert (sc.parallelize( Array((1, 11), (2, 22), (3, 33)) ).collect().head === (1, 11)) } + test("kryo with SerializableHyperLogLog") { + assert(sc.parallelize( Array(1, 2, 3, 2, 3, 3, 2, 3, 1) ).countApproxDistinct(0.01) === 3) + } + test("kryo with reduce") { val control = 1 :: 2 :: Nil val result = sc.parallelize(control, 2).map(new ClassWithoutNoArgConstructor(_)) @@ -206,6 +206,11 @@ to explicitly bump the version when building with YARN. It would be nice to figure out why Maven can't resolve this correctly (like SBT does). --> <dependency> + <groupId>com.clearspring.analytics</groupId> + <artifactId>stream</artifactId> + <version>2.4.0</version> + </dependency> + <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> <version>${protobuf.version}</version> diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 1df1abc9a3..b3b5fc788f 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -247,7 +247,8 @@ object SparkBuild extends Build { "com.codahale.metrics" % "metrics-ganglia" % "3.0.0", "com.codahale.metrics" % "metrics-graphite" % "3.0.0", "com.twitter" %% "chill" % "0.3.1", - "com.twitter" % "chill-java" % "0.3.1" + "com.twitter" % "chill-java" % "0.3.1", + "com.clearspring.analytics" % "stream" % "2.5.1" ) ) |