From e47e11720f7f51d720f98ad7dce959ac3ceef731 Mon Sep 17 00:00:00 2001 From: Matei Zaharia Date: Tue, 25 Sep 2012 15:46:18 -0700 Subject: Documentation updates --- docs/_plugins/copy_api_dirs.rb | 38 ++++++++-------- docs/css/main.css | 15 +++++-- docs/css/pygments-default.css | 100 ++++++++++++++++++++--------------------- docs/index.md | 41 ++++++++--------- 4 files changed, 103 insertions(+), 91 deletions(-) diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb index 7fb98851cd..e61c105449 100644 --- a/docs/_plugins/copy_api_dirs.rb +++ b/docs/_plugins/copy_api_dirs.rb @@ -1,28 +1,30 @@ require 'fileutils' include FileUtils -projects = ["core", "examples", "repl", "bagel"] +if ENV['SKIP_SCALADOC'] != '1' + projects = ["core", "examples", "repl", "bagel"] -puts "Moving to project root and building scaladoc." -curr_dir = pwd -cd("..") + puts "Moving to project root and building scaladoc." + curr_dir = pwd + cd("..") -puts "Running sbt/sbt doc from " + pwd + "; this may take a few minutes..." -puts `sbt/sbt doc` + puts "Running sbt/sbt doc from " + pwd + "; this may take a few minutes..." + puts `sbt/sbt doc` -puts "moving back into docs dir." -cd("docs") + puts "moving back into docs dir." + cd("docs") -# Copy over the scaladoc from each project into the docs directory. -# This directory will be copied over to _site when `jekyll` command is run. -projects.each do |project_name| - source = "../" + project_name + "/target/scala-2.9.2/api" - dest = "api/" + project_name + # Copy over the scaladoc from each project into the docs directory. + # This directory will be copied over to _site when `jekyll` command is run. + projects.each do |project_name| + source = "../" + project_name + "/target/scala-2.9.2/api" + dest = "api/" + project_name - puts "echo making directory " + dest - mkdir_p dest + puts "echo making directory " + dest + mkdir_p dest - # From the rubydoc: cp_r('src', 'dest') makes src/dest, but this doesn't. - puts "cp -r " + source + "/. " + dest - cp_r(source + "/.", dest) + # From the rubydoc: cp_r('src', 'dest') makes src/dest, but this doesn't. + puts "cp -r " + source + "/. " + dest + cp_r(source + "/.", dest) + end end diff --git a/docs/css/main.css b/docs/css/main.css index 263c8ef8ea..8310bc5859 100755 --- a/docs/css/main.css +++ b/docs/css/main.css @@ -10,8 +10,9 @@ } .navbar-inner { - padding-top: 5px; - padding-bottom: 5px; + padding-top: 6px; + padding-bottom: 6px; + font-size: 15px; } body #content { @@ -35,7 +36,15 @@ h3 { } code { - color: #333; + font-family: "Menlo", "Lucida Console", monospace; + background: white; + border: none; + padding: 0; + color: #902000; +} + +pre { + font-family: "Menlo", "Lucida Console", monospace; } .container { diff --git a/docs/css/pygments-default.css b/docs/css/pygments-default.css index f5815c25ca..6247cd8396 100644 --- a/docs/css/pygments-default.css +++ b/docs/css/pygments-default.css @@ -11,66 +11,66 @@ Also, I was thrown off for a while at first when I was using markdown code block inside my {% highlight scala %} ... {% endhighlight %} tags (I was using 4 spaces for this), when it turns out that pygments will insert the code (or pre?) tags for you. - */ + .hll { background-color: #ffffcc } -.c { color: #408080; font-style: italic } /* Comment */ -.err { border: 1px solid #FF0000 } /* Error */ -.k { color: #008000; font-weight: bold } /* Keyword */ +.c { color: #60a0b0; font-style: italic } /* Comment */ +.err { } /* Error */ +.k { color: #007020; font-weight: bold } /* Keyword */ .o { color: #666666 } /* Operator */ -.cm { color: #408080; font-style: italic } /* Comment.Multiline */ -.cp { color: #BC7A00 } /* Comment.Preproc */ -.c1 { color: #408080; font-style: italic } /* Comment.Single */ -.cs { color: #408080; font-style: italic } /* Comment.Special */ +.cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */ +.cp { color: #007020 } /* Comment.Preproc */ +.c1 { color: #60a0b0; font-style: italic } /* Comment.Single */ +.cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */ .gd { color: #A00000 } /* Generic.Deleted */ .ge { font-style: italic } /* Generic.Emph */ .gr { color: #FF0000 } /* Generic.Error */ .gh { color: #000080; font-weight: bold } /* Generic.Heading */ .gi { color: #00A000 } /* Generic.Inserted */ .go { color: #808080 } /* Generic.Output */ -.gp { color: #000080; font-weight: bold } /* Generic.Prompt */ +.gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ .gs { font-weight: bold } /* Generic.Strong */ .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ .gt { color: #0040D0 } /* Generic.Traceback */ -.kc { color: #008000; font-weight: bold } /* Keyword.Constant */ -.kd { color: #008000; font-weight: bold } /* Keyword.Declaration */ -.kn { color: #008000; font-weight: bold } /* Keyword.Namespace */ -.kp { color: #008000 } /* Keyword.Pseudo */ -.kr { color: #008000; font-weight: bold } /* Keyword.Reserved */ -.kt { color: #B00040 } /* Keyword.Type */ -.m { color: #666666 } /* Literal.Number */ -.s { color: #BA2121 } /* Literal.String */ -.na { color: #7D9029 } /* Name.Attribute */ -.nb { color: #008000 } /* Name.Builtin */ -.nc { color: #0000FF; font-weight: bold } /* Name.Class */ -.no { color: #880000 } /* Name.Constant */ -.nd { color: #AA22FF } /* Name.Decorator */ -.ni { color: #999999; font-weight: bold } /* Name.Entity */ -.ne { color: #D2413A; font-weight: bold } /* Name.Exception */ -.nf { color: #0000FF } /* Name.Function */ -.nl { color: #A0A000 } /* Name.Label */ -.nn { color: #0000FF; font-weight: bold } /* Name.Namespace */ -.nt { color: #008000; font-weight: bold } /* Name.Tag */ -.nv { color: #19177C } /* Name.Variable */ -.ow { color: #AA22FF; font-weight: bold } /* Operator.Word */ +.kc { color: #007020; font-weight: bold } /* Keyword.Constant */ +.kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ +.kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ +.kp { color: #007020 } /* Keyword.Pseudo */ +.kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ +.kt { color: #902000 } /* Keyword.Type */ +.m { color: #40a070 } /* Literal.Number */ +.s { color: #4070a0 } /* Literal.String */ +.na { color: #4070a0 } /* Name.Attribute */ +.nb { color: #007020 } /* Name.Builtin */ +.nc { color: #0e84b5; font-weight: bold } /* Name.Class */ +.no { color: #60add5 } /* Name.Constant */ +.nd { color: #555555; font-weight: bold } /* Name.Decorator */ +.ni { color: #d55537; font-weight: bold } /* Name.Entity */ +.ne { color: #007020 } /* Name.Exception */ +.nf { color: #06287e } /* Name.Function */ +.nl { color: #002070; font-weight: bold } /* Name.Label */ +.nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ +.nt { color: #062873; font-weight: bold } /* Name.Tag */ +.nv { color: #bb60d5 } /* Name.Variable */ +.ow { color: #007020; font-weight: bold } /* Operator.Word */ .w { color: #bbbbbb } /* Text.Whitespace */ -.mf { color: #666666 } /* Literal.Number.Float */ -.mh { color: #666666 } /* Literal.Number.Hex */ -.mi { color: #666666 } /* Literal.Number.Integer */ -.mo { color: #666666 } /* Literal.Number.Oct */ -.sb { color: #BA2121 } /* Literal.String.Backtick */ -.sc { color: #BA2121 } /* Literal.String.Char */ -.sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */ -.s2 { color: #BA2121 } /* Literal.String.Double */ -.se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */ -.sh { color: #BA2121 } /* Literal.String.Heredoc */ -.si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */ -.sx { color: #008000 } /* Literal.String.Other */ -.sr { color: #BB6688 } /* Literal.String.Regex */ -.s1 { color: #BA2121 } /* Literal.String.Single */ -.ss { color: #19177C } /* Literal.String.Symbol */ -.bp { color: #008000 } /* Name.Builtin.Pseudo */ -.vc { color: #19177C } /* Name.Variable.Class */ -.vg { color: #19177C } /* Name.Variable.Global */ -.vi { color: #19177C } /* Name.Variable.Instance */ -.il { color: #666666 } /* Literal.Number.Integer.Long */ +.mf { color: #40a070 } /* Literal.Number.Float */ +.mh { color: #40a070 } /* Literal.Number.Hex */ +.mi { color: #40a070 } /* Literal.Number.Integer */ +.mo { color: #40a070 } /* Literal.Number.Oct */ +.sb { color: #4070a0 } /* Literal.String.Backtick */ +.sc { color: #4070a0 } /* Literal.String.Char */ +.sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ +.s2 { color: #4070a0 } /* Literal.String.Double */ +.se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ +.sh { color: #4070a0 } /* Literal.String.Heredoc */ +.si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ +.sx { color: #c65d09 } /* Literal.String.Other */ +.sr { color: #235388 } /* Literal.String.Regex */ +.s1 { color: #4070a0 } /* Literal.String.Single */ +.ss { color: #517918 } /* Literal.String.Symbol */ +.bp { color: #007020 } /* Name.Builtin.Pseudo */ +.vc { color: #bb60d5 } /* Name.Variable.Class */ +.vg { color: #bb60d5 } /* Name.Variable.Global */ +.vi { color: #bb60d5 } /* Name.Variable.Instance */ +.il { color: #40a070 } /* Literal.Number.Integer.Long */ \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 69d55e505e..b9d9a8a36f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,7 +7,7 @@ title: Spark Overview TODO(andyk): Rewrite to make the Java API a first class part of the story. {% endcomment %} -Spark is a MapReduce-like cluster computing framework designed to support low-latency iterative jobs and interactive use from an interpreter. It is written in [Scala](http://www.scala-lang.org), a high-level language for the JVM, and exposes a clean language-integrated syntax that makes it easy to write parallel jobs. Spark runs on top of the [Apache Mesos](http://incubator.apache.org/mesos/) cluster manager, Hadoop YARN, or without an independent resource manager (i.e., in "standalone mode"). +Spark is a MapReduce-like cluster computing framework designed to support low-latency iterative jobs and interactive use from an interpreter. It exposes clean language-integrated APIs in [Scala](http://www.scala-lang.org) and Java, providing a wide array of parallel operations. Spark can run on top of the [Apache Mesos](http://incubator.apache.org/mesos/) cluster manager, Hadoop YARN, or without an independent resource manager ("standalone mode"). # Downloading @@ -15,19 +15,14 @@ Get Spark by checking out the master branch of the Git repository, using `git cl # Building -Spark requires [Scala 2.9](http://www.scala-lang.org/). -In addition, to run Spark on a cluster, you will need to install [Mesos](http://incubator.apache.org/mesos/), using the steps in -[Running Spark on Mesos]({{HOME_PATH}}running-on-mesos.html). However, if you just want to run Spark on a single machine (possibly using multiple cores), -you do not need Mesos. - -To build and run Spark, you will need to have Scala's `bin` directory in your `PATH`, +Spark requires [Scala 2.9.2](http://www.scala-lang.org/). You will need to have Scala's `bin` directory in your `PATH`, or you will need to set the `SCALA_HOME` environment variable to point to where you've installed Scala. Scala must be accessible through one -of these methods on Mesos slave nodes as well as on the master. +of these methods on slave nodes on your cluster. Spark uses [Simple Build Tool](https://github.com/harrah/xsbt/wiki), which is bundled with it. To compile the code, go into the top-level Spark directory and run - sbt/sbt compile + sbt/sbt package # Testing the Build @@ -44,7 +39,7 @@ thread, or `local[N]` to run locally with N threads. You should start by using ` Finally, Spark can be used interactively from a modified version of the Scala interpreter that you can start through `./spark-shell`. This is a great way to learn Spark. -# A Note About Hadoop +# A Note About Hadoop Versions Spark uses the Hadoop core library to talk to HDFS and other Hadoop-supported storage systems. Because the HDFS protocol has changed in different versions of @@ -54,23 +49,29 @@ of `project/SparkBuild.scala`, then rebuilding Spark (`sbt/sbt clean compile`). # Where to Go from Here -* [Spark Programming Guide]({{HOME_PATH}}scala-programming-guide.html): how to get started using Spark, and details on the API +Programming guides: +* [Spark Programming Guide]({{HOME_PATH}}scala-programming-guide.html): how to get started using Spark, and details on the Scala API +* [Java Programming Guide]({{HOME_PATH}}java-programming-guide.html): using Spark from Java + +Deployment guides: * [Running Spark on Amazon EC2]({{HOME_PATH}}ec2-scripts.html): scripts that let you launch a cluster on EC2 in about 5 minutes -* [Running Spark on Mesos]({{HOME_PATH}}running-on-mesos.html): instructions on how to deploy to a private cluster -* [Running Spark on YARN]({{HOME_PATH}}running-on-yarn.html): instructions on how to run Spark on top of a YARN cluster -* [Spark Standalone Mode]({{HOME_PATH}}spark-standalone.html): instructions on running Spark without Mesos -* [Configuration]({{HOME_PATH}}configuration.html): How to set up and customize Spark via its configuration system. -* [Bagel Programming Guide]({{HOME_PATH}}bagel-programming-guide.html): implementation of Google's Pregel on Spark -* [Spark Debugger]({{HOME_PATH}}spark-debugger.html): experimental work on a debugger for Spark jobs +* [Standalone Deploy Mode]({{HOME_PATH}}spark-standalone.html): launch a standalone cluster quickly without Mesos +* [Running Spark on Mesos]({{HOME_PATH}}running-on-mesos.html): deploy a private cluster using + [Apache Mesos](http://incubator.apache.org/mesos) +* [Running Spark on YARN]({{HOME_PATH}}running-on-yarn.html): deploy Spark on top of Hadoop NextGen (YARN) + +Miscellaneous: +* [Configuration]({{HOME_PATH}}configuration.html): customize Spark via its configuration system. +* [Bagel]({{HOME_PATH}}bagel-programming-guide.html): an implementation of Google's Pregel on Spark * [Contributing to Spark](contributing-to-spark.html) # Other Resources * [Spark Homepage](http://www.spark-project.org) -* [AMP Camp](http://ampcamp.berkeley.edu/) - In 2012, the AMP Lab hosted the first AMP Camp which featured talks and hands-on exercises about Spark, Shark, Mesos, and more. [Videos, slides](http://ampcamp.berkeley.edu/agenda) and the [exercises](http://ampcamp.berkeley.edu/exercises) are all available online now. Going through the videos and exercises is a great way to sharpen your Spark skills. +* [AMP Camp](http://ampcamp.berkeley.edu/) - In 2012, the AMP Lab hosted the first AMP Camp which featured talks and hands-on exercises about Spark, Shark, Mesos, and more. [Videos, slides](http://ampcamp.berkeley.edu/agenda) and the [exercises](http://ampcamp.berkeley.edu/exercises) are all available online, and provide a great introduction to Spark. * [Paper describing the programming model](http://www.cs.berkeley.edu/~matei/papers/2012/nsdi_spark.pdf) -* [Code Examples](http://spark-project.org/examples.html) (more also available in the [examples subfolder](https://github.com/mesos/spark/tree/master/examples/src/main/scala/spark/examples) of the Spark codebase) -* [Mailing List](http://groups.google.com/group/spark-users) +* [Code Examples](http://spark-project.org/examples.html): more are also available in the [examples subfolder](https://github.com/mesos/spark/tree/master/examples/src/main/scala/spark/examples) of Spark +* [Mailing List](http://groups.google.com/group/spark-users): ask here for help # Community -- cgit v1.2.3