diff options
59 files changed, 4861 insertions, 1 deletions
diff --git a/README.md b/README.md new file mode 100644 index 000000000..f02631a4c --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +Welcome to the Spark website. + +## Generating the website HTML + +In this directory you will find text files formatted using Markdown, with an ".md" suffix. + +You can generate the html version of these files by (installing and) running `jekyll build` (for newer versions of Jekyll) or just `jekyll` (for older versions) in this directory. + +The easiest way to install jekyll is via a Ruby Gem. This will create a directory called `site` containing index.html as well as the rest of the compiled directories and files. Read more about Jekyll at http://jekyllrb.com/docs + +In addition to generating the site as html from the markdown files, jekyll can serve the site via a webserver. To build and run a webserver use the command `jekyll serve` (or --serve for older ones) which runs the webserver on port 4000, then visit the site at http://localhost:4000. + +## Pygments + +We also use pygments (http://pygments.org) for syntax highlighting in documentation markdown pages. + +To mark a block of code in your markdown to be syntax highlighted by jekyll during the compile phase, use the following syntax: + + {% highlight scala %} + // Your scala code goes here, you can replace scala with many other + // supported languages too. + {% endhighlight %} + + You probably don't need to install that unless you want to regenerate the pygments css file. It requires Python, and can be installed by running `sudo easy_install Pygments`.
\ No newline at end of file diff --git a/_config.yml b/_config.yml new file mode 100644 index 000000000..82a24f02e --- /dev/null +++ b/_config.yml @@ -0,0 +1,12 @@ +pygments: true +markdown: kramdown +permalink: none +destination: site + +# The recommended way of viewing the website on your local machine is via jekyll using +# a webserver, e.g. with a command like: jekyll serve --watch --trace +# To compile the website such that it is viewable in a sane way via a file browser +# replace '/' here with the url that represents the website root dir in your file browser. +# E.g. on OS X this might be: +#url: file:///Users/andyk/Development/spark/website/site/ +url: / diff --git a/_includes/footer.html b/_includes/footer.html new file mode 100644 index 000000000..101cdba8e --- /dev/null +++ b/_includes/footer.html @@ -0,0 +1,6 @@ +<footer id="colophon" role="contentinfo"> + <div id="site-generator"> + <p>Spark is an open source project developed at the UC Berkeley <a href="https://amplab.cs.berkeley.edu">AMPLab</a>.</p> + <a class="amp-logo" style="background:url({{site.url}}images/amplab-small.png)" href="https://amplab.cs.berkeley.edu/" title="Brought to you by the UC Berkeley AMPLab." rel="generator"><!--Brought to you by the UC Berkeley AMPLab--> </a> + </div> +</footer><!-- #colophon --> diff --git a/_includes/navbar.html b/_includes/navbar.html new file mode 100644 index 000000000..c7e68fd58 --- /dev/null +++ b/_includes/navbar.html @@ -0,0 +1,23 @@ +<header id="branding" role="banner"> + <hgroup> + <h1 id="site-title"><span><a href="{{site.url}}" title="Spark" rel="home">Spark</a></span></h1> + <h2 id="site-description">Lightning-Fast Cluster Computing</h2> + </hgroup> + + <a href="{{site.url}}"> + <img src="{{site.url}}images/spark-project-header1.png" width="1000" height="220" alt="Spark: Lightning-Fast Cluster Computing" title="Spark: Lightning-Fast Cluster Computing" /> + </a> + + <nav id="access" role="navigation"> + <h3 class="assistive-text">Main menu</h3> + <div class="menu-main-menu-container"> + <ul id="menu-main-menu" class="menu"> + {% for p in site.navigation %} + <li class="menu-item menu-item-type-post_type menu-item-object-page {% if p.title == page.title %}current-menu-item{% endif %}"> + <a href="{{site.url}}{{p.url | replace:'/',''}}">{{p.title}}</a> + </li> + {% endfor %} + </ul></div> + </nav><!-- #access --> +</header><!-- #branding --> + diff --git a/_layouts/global.html b/_layouts/global.html new file mode 100644 index 000000000..937cbb911 --- /dev/null +++ b/_layouts/global.html @@ -0,0 +1,82 @@ +<!DOCTYPE html> +<!--[if IE 6]> +<html id="ie6" dir="ltr" lang="en-US"> +<![endif]--> +<!--[if IE 7]> +<html id="ie7" dir="ltr" lang="en-US"> +<![endif]--> +<!--[if IE 8]> +<html id="ie8" dir="ltr" lang="en-US"> +<![endif]--> +<!--[if !(IE 6) | !(IE 7) | !(IE 8) ]><!--> +<html dir="ltr" lang="en-US"> +<!--<![endif]--> +<head> + <link rel="shortcut icon" href="favicon.ico" /> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width" /> + <title>{{page.title}} | Spark</title> + + <link rel="stylesheet" type="text/css" media="all" href="{{site.url}}css/style.css" /> + <link rel="stylesheet" href="{{site.url}}css/pygments-default.css"> + + <script type="text/javascript">//<![CDATA[ + // Google Analytics for WordPress by Yoast v4.2.8 | http://yoast.com/wordpress/google-analytics/ + var _gaq = _gaq || []; + _gaq.push(['_setAccount', 'UA-32518208-1']); + _gaq.push(['_trackPageview']); + (function () { + var ga = document.createElement('script'); + ga.type = 'text/javascript'; + ga.async = true; + ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; + var s = document.getElementsByTagName('script')[0]; + s.parentNode.insertBefore(ga, s); + })(); + //]]></script> + + <link rel='canonical' href='{{site.url}}index.html' /> + + <style type="text/css"> + #site-title, + #site-description { + position: absolute !important; + clip: rect(1px 1px 1px 1px); /* IE6, IE7 */ + clip: rect(1px, 1px, 1px, 1px); + } + </style> + <style type="text/css" id="custom-background-css"> + body.custom-background { background-color: #f1f1f1; } + </style> +</head> + +<!--body class="page {% if page.sidebar %}two-column right-sidebar{% else %}singular{% endif%}"--> +<body class="{{page.type}}"> +<div id="page" class="hfeed"> + + {% include navbar.html %} + + <div id="main"> + <div id="primary"> + <div id="content" role="main"> + {% if page.type contains 'archive' %} + {{ content }} + {% else %} + <article class="page type-page status-publish hentry"> + {{ content }} + </article><!-- #post --> + {% endif %} + </div><!-- #content --> + {% if page.sidebar %} + <div id="secondary" class="widget-area" role="complementary"> + {{ page.sidebar | markdownify }} + </div> + {% endif %} + {% include footer.html %} + </div><!-- #primary --> + </div><!-- #main --> +</div><!-- #page --> + + +</body> +</html> diff --git a/_layouts/post.html b/_layouts/post.html new file mode 100644 index 000000000..f05db77dd --- /dev/null +++ b/_layouts/post.html @@ -0,0 +1,8 @@ +--- +layout: global +type: singular +--- +<h2>{{page.title}}</h2> + + +{{content}}
\ No newline at end of file diff --git a/_plugins/sidebar.rb b/_plugins/sidebar.rb new file mode 100644 index 000000000..9f652cb2d --- /dev/null +++ b/_plugins/sidebar.rb @@ -0,0 +1,16 @@ +module Liquid + class SidebarBlock < Liquid::Block + alias :super_render :render + + def initialize(tag_name, identifier, tokens) + super + end + + def render(context) + context.environments.first["page"]["sidebar"] = super_render(context) + '' + end + end +end +puts "registering new sidebar template" +Liquid::Template.register_tag('sidebar', Liquid::SidebarBlock)
\ No newline at end of file diff --git a/_plugins/site_navigation.rb b/_plugins/site_navigation.rb new file mode 100644 index 000000000..9296773d2 --- /dev/null +++ b/_plugins/site_navigation.rb @@ -0,0 +1,36 @@ +module Jekyll + + class SiteNavigation < Jekyll::Generator + safe true + priority :lowest + + def generate(site) + + # First remove all invisible items (default: nil = show in nav) + unsorted = [] + site.pages.each do |page| + if not page.data["navigation"].nil? + if not page.data["navigation"]["show"].nil? + unsorted << page if page.data["navigation"]["show"] != false + else + puts "no show under nav on " + page.data["title"] + end + else + puts "no nav on page " + page.data["title"] + end + end + + # Then sort em according to weight + sorted = unsorted.sort{ |a,b| a.data["navigation"]["weight"] <=> b.data["navigation"]["weight"] } + + # Debug info. + puts "Sorted resulting navigation: (use site.config['sorted_navigation']) " + sorted.each do |p| + puts p.inspect + end + + # Access this in Liquid using: site.navigation + site.config["navigation"] = sorted + end + end +end
\ No newline at end of file diff --git a/css/pygments-default.css b/css/pygments-default.css new file mode 100644 index 000000000..77d85d60c --- /dev/null +++ b/css/pygments-default.css @@ -0,0 +1,74 @@ +/* +This style sheet copied from Spark docs dir. +Documentation for pygments (and Jekyll for that matter) is super sparse. +To generate this, I had to run + `pygmentize -S default -f html > pygments-default.css` +But first I had to install pygments via easy_install pygments + +Also, I was thrown off for a while at first when I was using markdown +code block inside my {% highlight scala %} ... {% endhighlight %} tags +(I was using 4 spaces for this), when it turns out that pygments will +insert the code (or pre?) tags for you. +*/ + +.hll { background-color: #ffffcc } +.c { color: #60a0b0; font-style: italic } /* Comment */ +.err { } /* Error */ +.k { color: #007020; font-weight: bold } /* Keyword */ +.o { color: #666666 } /* Operator */ +.cm { color: #60a0b0; font-style: italic } /* Comment.Multiline */ +.cp { color: #007020 } /* Comment.Preproc */ +.c1 { color: #60a0b0; font-style: italic } /* Comment.Single */ +.cs { color: #60a0b0; background-color: #fff0f0 } /* Comment.Special */ +.gd { color: #A00000 } /* Generic.Deleted */ +.ge { font-style: italic } /* Generic.Emph */ +.gr { color: #FF0000 } /* Generic.Error */ +.gh { color: #000080; font-weight: bold } /* Generic.Heading */ +.gi { color: #00A000 } /* Generic.Inserted */ +.go { color: #808080 } /* Generic.Output */ +.gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ +.gs { font-weight: bold } /* Generic.Strong */ +.gu { color: #800080; font-weight: bold } /* Generic.Subheading */ +.gt { color: #0040D0 } /* Generic.Traceback */ +.kc { color: #007020; font-weight: bold } /* Keyword.Constant */ +.kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ +.kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ +.kp { color: #007020 } /* Keyword.Pseudo */ +.kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ +.kt { color: #902000 } /* Keyword.Type */ +.m { color: #40a070 } /* Literal.Number */ +.s { color: #4070a0 } /* Literal.String */ +.na { color: #4070a0 } /* Name.Attribute */ +.nb { color: #007020 } /* Name.Builtin */ +.nc { color: #0e84b5; font-weight: bold } /* Name.Class */ +.no { color: #60add5 } /* Name.Constant */ +.nd { color: #555555; font-weight: bold } /* Name.Decorator */ +.ni { color: #d55537; font-weight: bold } /* Name.Entity */ +.ne { color: #007020 } /* Name.Exception */ +.nf { color: #06287e } /* Name.Function */ +.nl { color: #002070; font-weight: bold } /* Name.Label */ +.nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ +.nt { color: #062873; font-weight: bold } /* Name.Tag */ +.nv { color: #bb60d5 } /* Name.Variable */ +.ow { color: #007020; font-weight: bold } /* Operator.Word */ +.w { color: #bbbbbb } /* Text.Whitespace */ +.mf { color: #40a070 } /* Literal.Number.Float */ +.mh { color: #40a070 } /* Literal.Number.Hex */ +.mi { color: #40a070 } /* Literal.Number.Integer */ +.mo { color: #40a070 } /* Literal.Number.Oct */ +.sb { color: #4070a0 } /* Literal.String.Backtick */ +.sc { color: #4070a0 } /* Literal.String.Char */ +.sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ +.s2 { color: #4070a0 } /* Literal.String.Double */ +.se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ +.sh { color: #4070a0 } /* Literal.String.Heredoc */ +.si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ +.sx { color: #c65d09 } /* Literal.String.Other */ +.sr { color: #235388 } /* Literal.String.Regex */ +.s1 { color: #4070a0 } /* Literal.String.Single */ +.ss { color: #517918 } /* Literal.String.Symbol */ +.bp { color: #007020 } /* Name.Builtin.Pseudo */ +.vc { color: #bb60d5 } /* Name.Variable.Class */ +.vg { color: #bb60d5 } /* Name.Variable.Global */ +.vi { color: #bb60d5 } /* Name.Variable.Instance */ +.il { color: #40a070 } /* Literal.Number.Integer.Long */
\ No newline at end of file diff --git a/css/style.css b/css/style.css new file mode 100644 index 000000000..f7651bdd3 --- /dev/null +++ b/css/style.css @@ -0,0 +1,3107 @@ +/* +Theme Name: Twenty Eleven +Theme URI: http://wordpress.org/extend/themes/twentyeleven +Author: the WordPress team +Author URI: http://wordpress.org/ +Description: The 2011 theme for WordPress is sophisticated, lightweight, and adaptable. Make it yours with a custom menu, header image, and background -- then go further with available theme options for light or dark color scheme, custom link colors, and three layout choices. Twenty Eleven comes equipped with a Showcase page template that transforms your front page into a showcase to show off your best content, widget support galore (sidebar, three footer areas, and a Showcase page widget area), and a custom "Ephemera" widget to display your Aside, Link, Quote, or Status posts. Included are styles for print and for the admin editor, support for featured images (as custom header images on posts and pages and as large images on featured "sticky" posts), and special styles for six different post formats. +Version: 1.4 +License: GNU General Public License v2 or later +License URI: http://www.gnu.org/licenses/gpl-2.0.html +Tags: dark, light, white, black, gray, one-column, two-columns, left-sidebar, right-sidebar, fixed-width, flexible-width, custom-background, custom-colors, custom-header, custom-menu, editor-style, featured-image-header, featured-images, full-width-template, microformats, post-formats, rtl-language-support, sticky-post, theme-options, translation-ready +Text Domain: twentyeleven +*/ + +/* =Reset default browser CSS. Based on work by Eric Meyer: http://meyerweb.com/eric/tools/css/reset/index.html +-------------------------------------------------------------- */ + +html, body, div, span, applet, object, iframe, +h1, h2, h3, h4, h5, h6, p, blockquote, pre, +a, abbr, acronym, address, big, cite, code, +del, dfn, em, font, ins, kbd, q, s, samp, +small, strike, strong, sub, sup, tt, var, +dl, dt, dd, ol, ul, li, +fieldset, form, label, legend, +table, caption, tbody, tfoot, thead, tr, th, td { + border: 0; + font-family: inherit; + font-size: 100%; + font-style: inherit; + font-weight: inherit; + margin: 0; + outline: 0; + padding: 0; + vertical-align: baseline; +} +:focus {/* remember to define focus styles! */ + outline: 0; +} +body { + background: #fff; + line-height: 1; +} +ol, ul { + list-style: none; +} +table {/* tables still need 'cellspacing="0"' in the markup */ + border-collapse: separate; + border-spacing: 0; +} +caption, th, td { + font-weight: normal; + text-align: left; +} +blockquote:before, blockquote:after, +q:before, q:after { + content: ""; +} +blockquote, q { + quotes: "" ""; +} +a img { + border: 0; +} +article, aside, details, figcaption, figure, +footer, header, hgroup, menu, nav, section { + display: block; +} + + +/* =Structure +----------------------------------------------- */ + +body { + padding: 0 2em; +} +#page { + margin: 2em auto; + max-width: 1000px; +} +#branding hgroup { + margin: 0 7.6%; +} +#access div { + margin: 0 7.6%; +} +#primary { + float: left; + margin: 0 -26.4% 0 0; + width: 100%; +} +#content { + margin: 0 34% 0 7.6%; + width: 58.4%; +} +#secondary { + float: right; + margin-right: 7.6%; + width: 18.8%; +} + +/* Singular */ +.singular #primary { + margin: 0; +} +.singular #content, +.left-sidebar.singular #content { + margin: 0 7.6%; + position: relative; + width: auto; +} +.singular .entry-header, +.singular .entry-content, +.singular footer.entry-meta, +.singular #comments-title { + margin: 0 auto; + width: 68.9%; +} + +/* Attachments */ +.singular .image-attachment .entry-content { + margin: 0 auto; + width: auto; +} +.singular .image-attachment .entry-description { + margin: 0 auto; + width: 68.9%; +} + +/* Showcase */ +.page-template-showcase-php #primary, +.left-sidebar.page-template-showcase-php #primary { + margin: 0; +} +.page-template-showcase-php #content, +.left-sidebar.page-template-showcase-php #content { + margin: 0 7.6%; + width: auto; +} +.page-template-showcase-php section.recent-posts { + float: right; + margin: 0 0 0 31%; + width: 69%; +} +.page-template-showcase-php #main .widget-area { + float: left; + margin: 0 -22.15% 0 0; + width: 22.15%; +} + +/* error404 */ +.error404 #primary { + float: none; + margin: 0; +} +.error404 #primary #content { + margin: 0 7.6%; + width: auto; +} + +/* Alignment */ +.alignleft { + display: inline; + float: left; + margin-right: 1.625em; +} +.alignright { + display: inline; + float: right; + margin-left: 1.625em; +} +.aligncenter { + clear: both; + display: block; + margin-left: auto; + margin-right: auto; +} + +/* Right Content */ +.left-sidebar #primary { + float: right; + margin: 0 0 0 -26.4%; + width: 100%; +} +.left-sidebar #content { + margin: 0 7.6% 0 34%; + width: 58.4%; +} +.left-sidebar #secondary { + float: left; + margin-left: 7.6%; + margin-right: 0; + width: 18.8%; +} + +/* One column */ +.one-column #page { + max-width: 690px; +} +.one-column #content { + margin: 0 7.6%; + width: auto; +} +.one-column #nav-below { + border-bottom: 1px solid #ddd; + margin-bottom: 1.625em; +} +.one-column #secondary { + float: none; + margin: 0 7.6%; + width: auto; +} +/* Simplify the showcase template */ +.one-column .page-template-showcase-php section.recent-posts { + float: none; + margin: 0; + width: 100%; +} +.one-column .page-template-showcase-php #main .widget-area { + float: none; + margin: 0; + width: auto; +} +.one-column .page-template-showcase-php .other-recent-posts { + border-bottom: 1px solid #ddd; +} +/* Simplify the showcase template when small feature */ +.one-column section.featured-post .attachment-small-feature { + border: none; + display: block; + height: auto; + max-width: 60%; + position: static; +} +.one-column article.feature-image.small { + margin: 0 0 1.625em; + padding: 0; +} +.one-column article.feature-image.small .entry-title { + font-size: 20px; + line-height: 1.3em; +} +.one-column article.feature-image.small .entry-summary { + height: 150px; + overflow: hidden; + padding: 0; + text-overflow: ellipsis; +} +.one-column article.feature-image.small .entry-summary a { + left: -9%; +} +/* Remove the margin on singular articles */ +.one-column.singular .entry-header, +.one-column.singular .entry-content, +.one-column.singular footer.entry-meta, +.one-column.singular #comments-title { + width: 100%; +} +/* Simplify the pullquotes and pull styles */ +.one-column.singular blockquote.pull { + margin: 0 0 1.625em; +} +.one-column.singular .pull.alignleft { + margin: 0 1.625em 0 0; +} +.one-column.singular .pull.alignright { + margin: 0 0 0 1.625em; +} +.one-column.singular .entry-meta .edit-link a { + position: absolute; + left: 0; + top: 40px; +} +.one-column.singular #author-info { + margin: 2.2em -8.8% 0; + padding: 20px 8.8%; +} +/* Make sure we have room for our comment avatars */ +.one-column .commentlist > li.comment { + margin-left: 102px; + width: auto; +} +/* Make sure the logo and search form don't collide */ +.one-column #branding #searchform { + right: 40px; + top: 4em; +} +/* Talking avatars take up too much room at this size */ +.one-column .commentlist > li.comment { + margin-left: 0; +} +.one-column .commentlist > li.comment .comment-meta, +.one-column .commentlist > li.comment .comment-content { + margin-right: 85px; +} +.one-column .commentlist .avatar { + background: transparent; + display: block; + padding: 0; + top: 1.625em; + left: auto; + right: 1.625em; +} +.one-column .commentlist .children .avatar { + background: none; + padding: 0; + position: absolute; + top: 2.2em; + left: 2.2em; +} +.one-column #respond { + width: auto; +} + + +/* =Global +----------------------------------------------- */ + +body, input, textarea { + color: #373737; + font: 15px "Helvetica Neue", Helvetica, Arial, sans-serif; + font-weight: 300; + line-height: 1.625; +} +body { + background: #e2e2e2; +} +#page { + background: #fff; +} + +/* Headings */ +h1,h2,h3,h4,h5,h6 { + clear: both; +} +hr { + background-color: #ccc; + border: 0; + height: 1px; + margin-bottom: 1.625em; +} + +/* Text elements */ +p { + margin-bottom: 1.625em; +} +ul, ol { + margin: 0 0 1.625em 2.5em; +} +ul { + list-style: square; +} +ol { + list-style-type: decimal; +} +ol ol { + list-style: upper-alpha; +} +ol ol ol { + list-style: lower-roman; +} +ol ol ol ol { + list-style: lower-alpha; +} +ul ul, ol ol, ul ol, ol ul { + margin-bottom: 0; +} +dl { + margin: 0 1.625em; +} +dt { + font-weight: bold; +} +dd { + margin-bottom: 1.625em; +} +strong { + font-weight: bold; +} +cite, em, i { + font-style: italic; +} +blockquote { + font-family: Georgia, "Bitstream Charter", serif; + font-style: italic; + font-weight: normal; + margin: 0 3em; +} +blockquote em, blockquote i, blockquote cite { + font-style: normal; +} +blockquote cite { + color: #666; + font: 12px "Helvetica Neue", Helvetica, Arial, sans-serif; + font-weight: 300; + letter-spacing: 0.05em; + text-transform: uppercase; +} +pre { + background: #f4f4f4; + font: 13px "Courier 10 Pitch", Courier, monospace; + line-height: 1.5; + margin-bottom: 1.625em; + overflow: auto; + padding: 0.75em 1.625em; +} +code, kbd, samp, var { + font: 13px Monaco, Consolas, "Andale Mono", "DejaVu Sans Mono", monospace; +} +abbr, acronym, dfn { + border-bottom: 1px dotted #666; + cursor: help; +} +address { + display: block; + margin: 0 0 1.625em; +} +ins { + background: #fff9c0; + text-decoration: none; +} +sup, +sub { + font-size: 10px; + height: 0; + line-height: 1; + position: relative; + vertical-align: baseline; +} +sup { + bottom: 1ex; +} +sub { + top: .5ex; +} + +/* Forms */ +input[type=text], +input[type=password], +textarea { + background: #fafafa; + -moz-box-shadow: inset 0 1px 1px rgba(0,0,0,0.1); + -webkit-box-shadow: inset 0 1px 1px rgba(0,0,0,0.1); + box-shadow: inset 0 1px 1px rgba(0,0,0,0.1); + border: 1px solid #ddd; + color: #888; +} +input[type=text]:focus, +textarea:focus { + color: #373737; +} +textarea { + padding-left: 3px; + width: 98%; +} +input[type=text] { + padding: 3px; +} +input#s { + background: url(images/search.png) no-repeat 5px 6px; + -moz-border-radius: 2px; + border-radius: 2px; + font-size: 14px; + height: 22px; + line-height: 1.2em; + padding: 4px 10px 4px 28px; +} +input#searchsubmit { + display: none; +} + +/* Links */ +a { + color: #1982d1; + text-decoration: none; +} +a:focus, +a:active, +a:hover { + text-decoration: underline; +} + +/* Assistive text */ +.assistive-text { + position: absolute !important; + clip: rect(1px 1px 1px 1px); /* IE6, IE7 */ + clip: rect(1px, 1px, 1px, 1px); +} +#access a.assistive-text:active, +#access a.assistive-text:focus { + background: #eee; + border-bottom: 1px solid #ddd; + color: #1982d1; + clip: auto !important; + font-size: 12px; + position: absolute; + text-decoration: underline; + top: 0; + left: 7.6%; +} + + +/* =Header +----------------------------------------------- */ + +#branding { + border-top: 2px solid #bbb; + padding-bottom: 10px; + position: relative; + z-index: 9999; +} +#site-title { + margin-right: 270px; + padding: 3.65625em 0 0; +} +#site-title a { + color: #111; + font-size: 30px; + font-weight: bold; + line-height: 36px; + text-decoration: none; +} +#site-title a:hover, +#site-title a:focus, +#site-title a:active { + color: #1982d1; +} +#site-description { + color: #7a7a7a; + font-size: 14px; + margin: 0 270px 3.65625em 0; +} +#branding img { + height: auto; + margin-bottom: -7px; + width: 100%; +} + + +/* =Menu +-------------------------------------------------------------- */ + +#access { + background: #222; /* Show a solid color for older browsers */ + background: -moz-linear-gradient(#252525, #0a0a0a); + background: -o-linear-gradient(#252525, #0a0a0a); + background: -webkit-gradient(linear, 0% 0%, 0% 100%, from(#252525), to(#0a0a0a)); /* older webkit syntax */ + background: -webkit-linear-gradient(#252525, #0a0a0a); + -webkit-box-shadow: rgba(0, 0, 0, 0.4) 0px 1px 2px; + -moz-box-shadow: rgba(0, 0, 0, 0.4) 0px 1px 2px; + box-shadow: rgba(0, 0, 0, 0.4) 0px 1px 2px; + clear: both; + display: block; + float: left; + margin: 0 auto 6px; + width: 100%; +} +#access ul { + font-size: 13px; + list-style: none; + margin: 0 0 0 -0.8125em; + padding-left: 0; +} +#access li { + float: left; + position: relative; +} +#access a { + color: #eee; + display: block; + line-height: 3.333em; + padding: 0 1.2125em; + text-decoration: none; +} +#access ul ul { + -moz-box-shadow: 0 3px 3px rgba(0,0,0,0.2); + -webkit-box-shadow: 0 3px 3px rgba(0,0,0,0.2); + box-shadow: 0 3px 3px rgba(0,0,0,0.2); + display: none; + float: left; + margin: 0; + position: absolute; + top: 3.333em; + left: 0; + width: 188px; + z-index: 99999; +} +#access ul ul ul { + left: 100%; + top: 0; +} +#access ul ul a { + background: #f9f9f9; + border-bottom: 1px dotted #ddd; + color: #444; + font-size: 13px; + font-weight: normal; + height: auto; + line-height: 1.4em; + padding: 10px 10px; + width: 168px; +} +#access li:hover > a, +#access ul ul :hover > a, +#access a:focus { + background: #efefef; +} +#access li:hover > a, +#access a:focus { + background: #f9f9f9; /* Show a solid color for older browsers */ + background: -moz-linear-gradient(#f9f9f9, #e5e5e5); + background: -o-linear-gradient(#f9f9f9, #e5e5e5); + background: -webkit-gradient(linear, 0% 0%, 0% 100%, from(#f9f9f9), to(#e5e5e5)); /* Older webkit syntax */ + background: -webkit-linear-gradient(#f9f9f9, #e5e5e5); + color: #373737; +} +#access ul li:hover > ul { + display: block; +} +#access .current-menu-item > a, +#access .current-menu-ancestor > a, +#access .current_page_item > a, +#access .current_page_ancestor > a { + font-weight: bold; +} + +/* Search Form */ +#branding #searchform { + position: absolute; + top: 3.8em; + right: 7.6%; + text-align: right; +} +#branding #searchform div { + margin: 0; +} +#branding #s { + float: right; + -webkit-transition-duration: 400ms; + -webkit-transition-property: width, background; + -webkit-transition-timing-function: ease; + -moz-transition-duration: 400ms; + -moz-transition-property: width, background; + -moz-transition-timing-function: ease; + -o-transition-duration: 400ms; + -o-transition-property: width, background; + -o-transition-timing-function: ease; + width: 72px; +} +#branding #s:focus { + background-color: #f9f9f9; + width: 196px; +} +#branding #searchsubmit { + display: none; +} +#branding .only-search #searchform { + top: 5px; + z-index: 1; +} +#branding .only-search #s { + background-color: #666; + border-color: #000; + color: #222; +} +#branding .only-search #s, +#branding .only-search #s:focus { + width: 85%; +} +#branding .only-search #s:focus { + background-color: #bbb; +} +#branding .with-image #searchform { + top: auto; + bottom: -27px; + max-width: 195px; +} +#branding .only-search + #access div { + padding-right: 205px; +} + + +/* =Content +----------------------------------------------- */ + +#main { + clear: both; + padding: 1.625em 0 0; +} +.page-title { + color: #666; + font-size: 10px; + font-weight: 500; + letter-spacing: 0.1em; + line-height: 2.6em; + margin: 0 0 2.6em; + text-transform: uppercase; +} +.page-title a { + font-size: 12px; + font-weight: bold; + letter-spacing: 0; + text-transform: none; +} +.hentry, +.no-results { + border-bottom: 1px solid #ddd; + margin: 0 0 1.625em; + padding: 0 0 1.625em; + position: relative; +} +.hentry:last-child, +.no-results { + border-bottom: none; +} +.blog .sticky .entry-header .entry-meta { + clip: rect(1px 1px 1px 1px); /* IE6, IE7 */ + clip: rect(1px, 1px, 1px, 1px); + position: absolute !important; +} +.entry-title, +.entry-header .entry-meta { + padding-right: 76px; +} +.entry-title { + clear: both; + color: #222; + font-size: 26px; + font-weight: bold; + line-height: 1.5em; + padding-bottom: .3em; + padding-top: 15px; +} +.entry-title, +.entry-title a { + color: #222; + text-decoration: none; +} +.entry-title a:hover, +.entry-title a:focus, +.entry-title a:active { + color: #1982d1; +} +.entry-meta { + color: #666; + clear: both; + font-size: 12px; + line-height: 18px; +} +.entry-meta a { + font-weight: bold; +} +.single-author .entry-meta .by-author { + display: none; +} +.entry-content, +.entry-summary { + padding: 1.625em 0 0; +} +.entry-content h1, +.entry-content h2, +.comment-content h1, +.comment-content h2 { + color: #000; + font-weight: bold; + margin: 0 0 .8125em; +} +.entry-content h3, +.comment-content h3 { + font-size: 10px; + letter-spacing: 0.1em; + line-height: 2.6em; + text-transform: uppercase; +} +.entry-content table, +.comment-content table { + border-bottom: 1px solid #ddd; + margin: 0 0 1.625em; + width: 100%; +} +.entry-content th, +.comment-content th { + color: #666; + font-size: 10px; + font-weight: 500; + letter-spacing: 0.1em; + line-height: 2.6em; + text-transform: uppercase; +} +.entry-content td, +.comment-content td { + border-top: 1px solid #ddd; + padding: 6px 10px 6px 0; +} +.entry-content #s { + width: 75%; +} +.comment-content ul, +.comment-content ol { + margin-bottom: 1.625em; +} +.comment-content ul ul, +.comment-content ol ol, +.comment-content ul ol, +.comment-content ol ul { + margin-bottom: 0; +} +dl.gallery-item { + margin: 0; +} +.page-link { + clear: both; + display: block; + margin: 0 0 1.625em; +} +.page-link a { + background: #eee; + color: #373737; + margin: 0; + padding: 2px 3px; + text-decoration: none; +} +.page-link a:hover { + background: #888; + color: #fff; + font-weight: bold; +} +.page-link span { + margin-right: 6px; +} +.entry-meta .edit-link a, +.commentlist .edit-link a { + background: #eee; + -moz-border-radius: 3px; + border-radius: 3px; + color: #666; + float: right; + font-size: 12px; + line-height: 1.5em; + font-weight: 300; + text-decoration: none; + padding: 0 8px; +} +.entry-meta .edit-link a:hover, +.commentlist .edit-link a:hover { + background: #888; + color: #fff; +} +.entry-content .edit-link { + clear: both; + display: block; +} + +/* Images */ +.entry-content img, +.comment-content img, +.widget img { + max-width: 97.5%; /* Fluid images for posts, comments, and widgets */ +} +img[class*="align"], +img[class*="wp-image-"], +img[class*="attachment-"] { + height: auto; /* Make sure images with WordPress-added height and width attributes are scaled correctly */ +} +img.size-full, +img.size-large { + max-width: 97.5%; + width: auto; /* Prevent stretching of full-size and large-size images with height and width attributes in IE8 */ + height: auto; /* Make sure images with WordPress-added height and width attributes are scaled correctly */ +} +.entry-content img.wp-smiley { + border: none; + margin-bottom: 0; + margin-top: 0; + padding: 0; +} +img.alignleft, +img.alignright, +img.aligncenter { + margin-bottom: 1.625em; +} +p img, +.wp-caption { + margin-top: 0.4em; +} +.wp-caption { + background: #eee; + margin-bottom: 1.625em; + max-width: 96%; + padding: 9px; +} +.wp-caption img { + display: block; + margin: 0 auto; + max-width: 98%; +} +.wp-caption .wp-caption-text, +.gallery-caption { + color: #666; + font-family: Georgia, serif; + font-size: 12px; +} +.wp-caption .wp-caption-text { + margin-bottom: 0.6em; + padding: 10px 0 5px 40px; + position: relative; +} +.wp-caption .wp-caption-text:before { + color: #666; + content: '\2014'; + font-size: 14px; + font-style: normal; + font-weight: bold; + margin-right: 5px; + position: absolute; + left: 10px; + top: 7px; +} +#content .gallery { + margin: 0 auto 1.625em; +} +#content .gallery a img { + border: none; +} +img#wpstats { + display: block; + margin: 0 auto 1.625em; +} +#content .gallery-columns-4 .gallery-item { + width: 23%; + padding-right: 2%; +} +#content .gallery-columns-4 .gallery-item img { + width: 100%; + height: auto; +} + +/* Image borders */ +img[class*="align"], +img[class*="wp-image-"], +#content .gallery .gallery-icon img {/* Add fancy borders to all WordPress-added images but not things like badges and icons and the like */ + border: 1px solid #ddd; + padding: 6px; +} +.wp-caption img { + border-color: #eee; +} +a:focus img[class*="align"], +a:hover img[class*="align"], +a:active img[class*="align"], +a:focus img[class*="wp-image-"], +a:hover img[class*="wp-image-"], +a:active img[class*="wp-image-"], +#content .gallery .gallery-icon a:focus img, +#content .gallery .gallery-icon a:hover img, +#content .gallery .gallery-icon a:active img {/* Add some useful style to those fancy borders for linked images ... */ + background: #eee; + border-color: #bbb; +} +.wp-caption a:focus img, +.wp-caption a:active img, +.wp-caption a:hover img {/* ... including captioned images! */ + background: #fff; + border-color: #ddd; +} + +/* Make sure embeds and iframes fit their containers */ +embed, +iframe, +object { + max-width: 100%; +} + +/* Password Protected Posts */ +.post-password-required .entry-header .comments-link { + margin: 1.625em 0 0; +} +.post-password-required input[type=password] { + margin: 0.8125em 0; +} +.post-password-required input[type=password]:focus { + background: #f7f7f7; +} + +/* Author Info */ +#author-info { + font-size: 12px; + overflow: hidden; +} +.singular #author-info { + background: #f9f9f9; + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; + margin: 2.2em -35.6% 0 -35.4%; + padding: 20px 35.4%; +} +.archive #author-info { + border-bottom: 1px solid #ddd; + margin: 0 0 2.2em; + padding: 0 0 2.2em; +} +#author-avatar { + float: left; + margin-right: -78px; +} +#author-avatar img { + background: #fff; + -moz-border-radius: 3px; + border-radius: 3px; + -webkit-box-shadow: 0 1px 2px #bbb; + -moz-box-shadow: 0 1px 2px #bbb; + box-shadow: 0 1px 2px #bbb; + padding: 3px; +} +#author-description { + float: left; + margin-left: 108px; +} +#author-description h2 { + color: #000; + font-size: 15px; + font-weight: bold; + margin: 5px 0 10px; +} + +/* Comments link */ +.entry-header .comments-link a { + background: #eee url(images/comment-bubble.png) no-repeat; + color: #666; + font-size: 13px; + font-weight: normal; + line-height: 35px; + overflow: hidden; + padding: 0 0 0; + position: absolute; + top: 1.5em; + right: 0; + text-align: center; + text-decoration: none; + width: 43px; + height: 36px; +} +.entry-header .comments-link a:hover, +.entry-header .comments-link a:focus, +.entry-header .comments-link a:active { + background-color: #1982d1; + color: #fff; + color: rgba(255,255,255,0.8); +} +.entry-header .comments-link .leave-reply { + visibility: hidden; +} + +/* +Post Formats Headings +To hide the headings, display: none the ".entry-header .entry-format" selector, +and remove the padding rules below. +*/ +.entry-header .entry-format { + color: #666; + font-size: 10px; + font-weight: 500; + letter-spacing: 0.1em; + line-height: 2.6em; + position: absolute; + text-transform: uppercase; + top: -5px; +} +.entry-header hgroup .entry-title { + padding-top: 15px; +} +article.format-aside .entry-content, +article.format-link .entry-content, +article.format-status .entry-content { + padding: 20px 0 0; +} +article.format-status .entry-content { + min-height: 65px; +} +.recent-posts .entry-header .entry-format { + display: none; +} +.recent-posts .entry-header hgroup .entry-title { + padding-top: 0; +} + +/* Singular content styles for Posts and Pages */ +.singular .hentry { + border-bottom: none; + padding: 4.875em 0 0; + position: relative; +} +.singular.page .hentry { + padding: 3.5em 0 0; +} +.singular .entry-title { + color: #000; + font-size: 36px; + font-weight: bold; + line-height: 48px; +} +.singular .entry-title, +.singular .entry-header .entry-meta { + padding-right: 0; +} +.singular .entry-header .entry-meta { + position: absolute; + top: 0; + left: 0; +} +blockquote.pull { + font-size: 21px; + font-weight: bold; + line-height: 1.6125em; + margin: 0 0 1.625em; + text-align: center; +} +.singular blockquote.pull { + margin: 0 -22.25% 1.625em; +} +.pull.alignleft { + margin: 0 1.625em 0 0; + text-align: right; +} +.singular .pull.alignleft { + margin: 0 1.625em 0 -22.25%; +} +.pull.alignright { + margin: 0 0 0 1.625em; + text-align: left; +} +blockquote.pull.alignleft, +blockquote.pull.alignright { + width: 33%; +} +.singular .pull.alignright { + margin: 0 -22.25% 0 1.625em; +} +.singular blockquote.pull.alignleft, +.singular blockquote.pull.alignright { + width: 33%; +} +.singular .entry-meta .edit-link a { + bottom: auto; + left: 50px; + position: absolute; + right: auto; + top: 80px; +} + + +/* =Aside +----------------------------------------------- */ + +.format-aside .entry-title, +.format-aside .entry-header .comments-link { + display: none; +} +.singular .format-aside .entry-title { + display: block; +} +.format-aside .entry-content { + padding: 0; +} +.singular .format-aside .entry-content { + padding: 1.625em 0 0; +} + + +/* =Link +----------------------------------------------- */ + +.format-link .entry-title, +.format-link .entry-header .comments-link { + display: none; +} +.singular .format-link .entry-title { + display: block; +} +.format-link .entry-content { + padding: 0; +} +.singular .format-link .entry-content { + padding: 1.625em 0 0; +} + + +/* =Gallery +----------------------------------------------- */ + +.format-gallery .gallery-thumb { + float: left; + display: block; + margin: .375em 1.625em 0 0; +} + + +/* =Status +----------------------------------------------- */ + +.format-status .entry-title, +.format-status .entry-header .comments-link { + display: none; +} +.singular .format-status .entry-title { + display: block; +} +.format-status .entry-content { + padding: 0; +} +.singular .format-status .entry-content { + padding: 1.625em 0 0; +} +.format-status img.avatar { + -moz-border-radius: 3px; + border-radius: 3px; + -webkit-box-shadow: 0 1px 2px #ccc; + -moz-box-shadow: 0 1px 2px #ccc; + box-shadow: 0 1px 2px #ccc; + float: left; + margin: 4px 10px 2px 0; + padding: 0; +} + + +/* =Quote +----------------------------------------------- */ + +.format-quote blockquote { + color: #555; + font-size: 17px; + margin: 0; +} + + +/* =Image +----------------------------------------------- */ + +.indexed.format-image .entry-header { + min-height: 61px; /* Prevent the comment icon from colliding with the image when there is no title */ +} +.indexed.format-image .entry-content { + padding-top: 0.5em; +} +.indexed.format-image .entry-content p { + margin: 1em 0; +} +.indexed.format-image .entry-content p:first-child, +.indexed.format-image .entry-content p:first-child a, +.indexed.format-image .entry-content p:first-child img { + display: block; + margin: 0; +} +.indexed.format-image .entry-content .wp-caption .wp-caption-text { + margin: 0; + padding-bottom: 1em; +} +.indexed.format-image footer.entry-meta { + background: #ddd; + overflow: hidden; + padding: 4%; + max-width: 96%; +} +.indexed.format-image div.entry-meta { + display: inline-block; + float: left; + width: 35%; +} +.indexed.format-image div.entry-meta + div.entry-meta { + float: none; + width: 65%; +} +.indexed.format-image .entry-meta span.cat-links, +.indexed.format-image .entry-meta span.tag-links, +.indexed.format-image .entry-meta span.comments-link { + display: block; +} +.indexed.format-image footer.entry-meta a { + color: #444; +} +.indexed.format-image footer.entry-meta a:hover { + color: #fff; +} +#content .indexed.format-image img { + border: none; + max-width: 100%; + padding: 0; +} +.indexed.format-image .wp-caption { + background: #111; + margin-bottom: 0; + max-width: 96%; + padding: 2% 2% 0; +} +.indexed.format-image .wp-caption .wp-caption-text { + color: #ddd; +} +.indexed.format-image .wp-caption .wp-caption-text:before { + color: #444; +} +.indexed.format-image a:hover img { + opacity: 0.8; +} + + +/* =error404 +----------------------------------------------- */ + +.error404 #main #searchform { + background: #f9f9f9; + border: 1px solid #ddd; + border-width: 1px 0; + margin: 0 -8.9% 1.625em; + overflow: hidden; + padding: 1.625em 8.9%; +} +.error404 #main #s { + width: 95%; +} +.error404 #main .widget { + clear: none; + float: left; + margin-right: 3.7%; + width: 30.85%; +} +.error404 #main .widget_archive { + margin-right: 0; +} +.error404 #main .widget_tag_cloud { + float: none; + margin-right: 0; + width: 100%; +} +.error404 .widgettitle { + font-size: 10px; + letter-spacing: 0.1em; + line-height: 2.6em; + text-transform: uppercase; +} + + +/* =Showcase +----------------------------------------------- */ + +h1.showcase-heading { + color: #666; + font-size: 10px; + font-weight: 500; + letter-spacing: 0.1em; + line-height: 2.6em; + text-transform: uppercase; +} + +/* Intro */ +article.intro { + background: #f9f9f9; + border-bottom: none; + margin: -1.855em -8.9% 1.625em; + padding: 0 8.9%; +} +article.intro .entry-title { + display: none; +} +article.intro .entry-content { + color: #111; + font-size: 16px; + padding: 1.625em 0 0.625em; +} +article.intro .edit-link a { + background: #aaa; + -moz-border-radius: 3px; + border-radius: 3px; + color: #fff; + font-size: 12px; + padding: 0 8px; + position: absolute; + top: 30px; + right: 20px; + text-decoration: none; +} +article.intro .edit-link a:hover, +article.intro .edit-link a:focus, +article.intro .edit-link a:active { + background: #777; +} + +/* Featured post */ +section.featured-post { + float: left; + margin: -1.625em -8.9% 1.625em; + padding: 1.625em 8.9% 0; + position: relative; + width: 100%; +} +section.featured-post .hentry { + border: none; + color: #666; + margin: 0; +} +section.featured-post .entry-meta { + clip: rect(1px 1px 1px 1px); /* IE6, IE7 */ + clip: rect(1px, 1px, 1px, 1px); + position: absolute !important; +} + +/* Small featured post */ +section.featured-post .attachment-small-feature { + float: right; + height: auto; + margin: 0 -8.9% 1.625em 0; + max-width: 59%; + position: relative; + right: -15px; +} +section.featured-post.small { + padding-top: 0; +} +section.featured-post .attachment-small-feature:hover, +section.featured-post .attachment-small-feature:focus, +section.featured-post .attachment-small-feature:active { + opacity: .8; +} +article.feature-image.small { + float: left; + margin: 0 0 1.625em; + width: 45%; +} +article.feature-image.small .entry-title { + line-height: 1.2em; +} +article.feature-image.small .entry-summary { + color: #555; + font-size: 13px; +} +article.feature-image.small .entry-summary p a { + background: #222; + color: #eee; + display: block; + left: -23.8%; + padding: 9px 26px 9px 85px; + position: relative; + text-decoration: none; + top: 20px; + width: 180px; + z-index: 1; +} +article.feature-image.small .entry-summary p a:hover { + background: #1982d1; + color: #eee; + color: rgba(255,255,255,0.8); +} + +/* Large featured post */ +section.feature-image.large { + border: none; + max-height: 288px; + padding: 0; + width: 100%; +} +section.feature-image.large .showcase-heading { + display: none; +} +section.feature-image.large .hentry { + border-bottom: none; + left: 9%; + margin: 1.625em 9% 0 0; + position: absolute; + top: 0; +} +article.feature-image.large .entry-title a { + background: #222; + background: rgba(0,0,0,0.8); + -moz-border-radius: 3px; + border-radius: 3px; + color: #fff; + display: inline-block; + font-weight: 300; + padding: .2em 20px; +} +section.feature-image.large:hover .entry-title a, +section.feature-image.large .entry-title:hover a { + background: #eee; + background: rgba(255,255,255,0.8); + color: #222; +} +article.feature-image.large .entry-summary { + display: none; +} +section.feature-image.large img { + display: block; + height: auto; + max-width: 117.9%; + padding: 0 0 6px; +} + +/* Featured Slider */ +.featured-posts { + border-bottom: 1px solid #ddd; + display: block; + height: 328px; + margin: 1.625em -8.9% 20px; + max-width: 1000px; + padding: 0; + position: relative; + overflow: hidden; +} +.featured-posts .showcase-heading { + padding-left: 8.9%; +} +.featured-posts section.featured-post { + background: #fff; + height: 288px; + left: 0; + margin: 0; + position: absolute; + top: 30px; + width: auto; +} +.featured-posts section.featured-post.large { + max-width: 100%; + overflow: hidden; +} +.featured-posts section.featured-post { + -webkit-transition-duration: 200ms; + -webkit-transition-property: opacity, visibility; + -webkit-transition-timing-function: ease; + -moz-transition-duration: 200ms; + -moz-transition-property: opacity, visibility; + -moz-transition-timing-function: ease; +} +.featured-posts section.featured-post { + opacity: 0; + visibility: hidden; +} +.featured-posts #featured-post-1 { + opacity: 1; + visibility: visible; +} +.featured-post .feature-text:after, +.featured-post .feature-image.small:after { + content: ' '; + background: -moz-linear-gradient(top, rgba(255,255,255,0) 0%, rgba(255,255,255,1) 100%); /* FF3.6+ */ + background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,rgba(255,255,255,0)), color-stop(100%,rgba(255,255,255,1))); /* Chrome,Safari4+ */ + background: -webkit-linear-gradient(top, rgba(255,255,255,0) 0%,rgba(255,255,255,1) 100%); /* Chrome10+,Safari5.1+ */ + background: -o-linear-gradient(top, rgba(255,255,255,0) 0%,rgba(255,255,255,1) 100%); /* Opera11.10+ */ + background: -ms-linear-gradient(top, rgba(255,255,255,0) 0%,rgba(255,255,255,1) 100%); /* IE10+ */ + filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#00ffffff', endColorstr='#ffffff',GradientType=0 ); /* IE6-9 */ + background: linear-gradient(top, rgba(255,255,255,0) 0%,rgba(255,255,255,1) 100%); /* W3C */ + width: 100%; + height: 45px; + position: absolute; + top: 230px; +} +.featured-post .feature-image.small:after { + top: 253px; +} +#content .feature-slider { + top: 5px; + right: 8.9%; + overflow: visible; + position: absolute; +} +.feature-slider ul { + list-style-type: none; + margin: 0; +} +.feature-slider li { + float: left; + margin: 0 6px; +} +.feature-slider a { + background: #3c3c3c; + background: rgba(60,60,60,0.9); + -moz-border-radius: 12px; + border-radius: 12px; + -webkit-box-shadow: inset 1px 1px 5px rgba(0,0,0,0.5), inset 0 0 2px rgba(255,255,255,0.5); + -moz-box-shadow: inset 1px 1px 5px rgba(0,0,0,0.5), inset 0 0 2px rgba(255,255,255,0.5); + box-shadow: inset 1px 1px 5px rgba(0,0,0,0.5), inset 0 0 2px rgba(255,255,255,0.5); + display: block; + width: 14px; + height: 14px; +} +.feature-slider a.active { + background: #1982d1; + -webkit-box-shadow: inset 1px 1px 5px rgba(0,0,0,0.4), inset 0 0 2px rgba(255,255,255,0.8); + -moz-box-shadow: inset 1px 1px 5px rgba(0,0,0,0.4), inset 0 0 2px rgba(255,255,255,0.8); + box-shadow: inset 1px 1px 5px rgba(0,0,0,0.4), inset 0 0 2px rgba(255,255,255,0.8); + cursor: default; + opacity: 0.5; +} + +/* Recent Posts */ +section.recent-posts { + padding: 0 0 1.625em; +} +section.recent-posts .hentry { + border: none; + margin: 0; +} +section.recent-posts .other-recent-posts { + border-bottom: 1px solid #ddd; + list-style: none; + margin: 0; +} +section.recent-posts .other-recent-posts li { + padding: 0.3125em 0; + position: relative; +} +section.recent-posts .other-recent-posts .entry-title { + border-top: 1px solid #ddd; + font-size: 17px; +} +section.recent-posts .other-recent-posts a[rel="bookmark"] { + color: #373737; + float: left; + max-width: 84%; +} +section.recent-posts .other-recent-posts a[rel="bookmark"]:after { + content: '-'; + color: transparent; + font-size: 11px; +} +section.recent-posts .other-recent-posts a[rel="bookmark"]:hover { +} +section.recent-posts .other-recent-posts .comments-link a, +section.recent-posts .other-recent-posts .comments-link > span { + border-bottom: 2px solid #999; + bottom: -2px; + color: #444; + display: block; + font-size: 10px; + font-weight: 500; + line-height: 2.76333em; + padding: 0.3125em 0 0.3125em 1em; + position: absolute; + right: 0; + text-align: right; + text-transform: uppercase; + z-index: 1; +} +section.recent-posts .other-recent-posts .comments-link > span { + border-color: #bbb; + color: #888; +} +section.recent-posts .other-recent-posts .comments-link a:hover { + color: #1982d1; + border-color: #1982d1; +} +section.recent-posts .other-recent-posts li:after { + clear: both; + content: '.'; + display: block; + height: 0; + visibility: hidden; +} + + +/* =Attachments +----------------------------------------------- */ + +.image-attachment div.attachment { + background: #f9f9f9; + border: 1px solid #ddd; + border-width: 1px 0; + margin: 0 -8.9% 1.625em; + overflow: hidden; + padding: 1.625em 1.625em 0; + text-align: center; +} +.image-attachment div.attachment img { + display: block; + height: auto; + margin: 0 auto 1.625em; + max-width: 100%; +} +.image-attachment div.attachment a img { + border-color: #f9f9f9; +} +.image-attachment div.attachment a:focus img, +.image-attachment div.attachment a:hover img, +.image-attachment div.attachment a:active img { + border-color: #ddd; + background: #fff; +} +.image-attachment .entry-caption p { + font-size: 10px; + letter-spacing: 0.1em; + line-height: 2.6em; + margin: 0 0 2.6em; + text-transform: uppercase; +} + + +/* =Navigation +-------------------------------------------------------------- */ + +#content nav { + clear: both; + overflow: hidden; + padding: 0 0 1.625em; +} +#content nav a { + font-size: 12px; + font-weight: bold; + line-height: 2.2em; +} +#nav-above { + padding: 0 0 1.625em; +} +#nav-above { + display: none; +} +.paged #nav-above { + display: block; +} +.nav-previous { + float: left; + width: 50%; +} +.nav-next { + float: right; + text-align: right; + width: 50%; +} +#content nav .meta-nav { + font-weight: normal; +} + +/* Singular navigation */ +#nav-single { + float: right; + position: relative; + top: -0.3em; + text-align: right; + z-index: 1; +} +#nav-single .nav-previous, +#nav-single .nav-next { + width: auto; +} +#nav-single .nav-next { + padding-left: .5em; +} +#nav-single .nav-previous { + padding-right: .5em; +} + + +/* =Widgets +----------------------------------------------- */ + +.widget-area { + font-size: 12px; +} +.widget { + clear: both; + margin: 0 0 2.2em; +} +.widget-title { + color: #666; + font-size: 10px; + font-weight: 500; + letter-spacing: 0.1em; + line-height: 2.6em; + text-transform: uppercase; +} +.widget ul { + font-size: 15px; + margin: 0; +} +.widget ul ul { + margin-left: 1.5em; +} +.widget ul li { + color: #777; + font-size: 13px; +} +.widget a { + font-weight: bold; + text-decoration: none; +} +.widget a:hover, +.widget a:focus, +.widget a:active { + text-decoration: underline; +} + +/* Search Widget */ +.widget_search form { + margin: 0 0 1.625em; +} +.widget_search #s { + width: 77%; +} +.widget_search #searchsubmit { + background: #ddd; + border: 1px solid #ccc; + -webkit-box-shadow: inset 0px -1px 1px rgba(0, 0, 0, 0.09); + -moz-box-shadow: inset 0px -1px 1px rgba(0, 0, 0, 0.09); + box-shadow: inset 0px -1px 1px rgba(0, 0, 0, 0.09); + color: #888; + font-size: 13px; + line-height: 25px; + position: relative; + top: -2px; +} +.widget_search #searchsubmit:active { + background: #1982d1; + border-color: #0861a5; + -webkit-box-shadow: inset 0px 1px 1px rgba(0, 0, 0, 0.1); + -moz-box-shadow: inset 0px 1px 1px rgba(0, 0, 0, 0.1); + box-shadow: inset 0px 1px 1px rgba(0, 0, 0, 0.1); + color: #bfddf3; +} + +/* Ephemera Widget */ +section.ephemera ol, +.widget_twentyeleven_ephemera ol { + list-style: square; + margin: 5px 0 0; +} +.widget_twentyeleven_ephemera .widget-entry-title { + font-size: 15px; + font-weight: bold; + padding: 0; +} +.widget_twentyeleven_ephemera .comments-link a, +.widget_twentyeleven_ephemera .comments-link > span { + color: #666; + display: block; + font-size: 10px; + font-weight: 500; + line-height: 2.76333em; + text-transform: uppercase; +} +section.ephemera .entry-title .comments-link a:hover, +.widget_twentyeleven_ephemera .entry-title .comments-link a:hover { +} +section.ephemera .entry-title a span { + color: #29628d; +} + +/* Twitter */ +.widget_twitter li { + list-style-type: none; + margin-bottom: 14px; +} +.widget_twitter .timesince { + display: block; + font-size: 11px; + margin-right: -10px; + text-align: right; +} + +/* Widget Image */ +.widget_image img { + border: 0; + padding: 0; + height: auto; + max-width: 100%; +} + +/* Calendar Widget */ + +.widget_calendar #wp-calendar { + color: #555; + width: 95%; + text-align: center; +} +.widget_calendar #wp-calendar caption, +.widget_calendar #wp-calendar td, +.widget_calendar #wp-calendar th { + text-align: center; +} +.widget_calendar #wp-calendar caption { + font-size: 11px; + font-weight: 500; + padding: 5px 0 3px 0; + text-transform: uppercase; +} +.widget_calendar #wp-calendar th { + background: #f4f4f4; + border-top: 1px solid #ccc; + border-bottom: 1px solid #ccc; + font-weight: bold; +} +.widget_calendar #wp-calendar tfoot td { + background: #f4f4f4; + border-top: 1px solid #ccc; + border-bottom: 1px solid #ccc; +} + + +/* =Comments +----------------------------------------------- */ + +#comments-title { + color: #666; + font-size: 10px; + font-weight: 500; + line-height: 2.6em; + padding: 0 0 2.6em; + text-transform: uppercase; +} +.nopassword, +.nocomments { + color: #aaa; + font-size: 24px; + font-weight: 100; + margin: 26px 0; + text-align: center; +} +.commentlist { + list-style: none; + margin: 0 auto; + width: 68.9%; +} +.content .commentlist, +.page-template-sidebar-page-php .commentlist { + width: 100%; /* reset the width for the one-column and sidebar page layout */ +} +.commentlist > li.comment { + background: #f6f6f6; + border: 1px solid #ddd; + -moz-border-radius: 3px; + border-radius: 3px; + margin: 0 0 1.625em; + padding: 1.625em; + position: relative; +} +.commentlist .pingback { + margin: 0 0 1.625em; + padding: 0 1.625em; +} +.commentlist .children { + list-style: none; + margin: 0; +} +.commentlist .children li.comment { + background: #fff; + border-left: 1px solid #ddd; + -moz-border-radius: 0 3px 3px 0; + border-radius: 0 3px 3px 0; + margin: 1.625em 0 0; + padding: 1.625em; + position: relative; +} +.commentlist .children li.comment .fn { + display: block; +} +.comment-meta .fn { + font-style: normal; +} +.comment-meta { + color: #666; + font-size: 12px; + line-height: 2.2em; +} +.commentlist .children li.comment .comment-meta { + line-height: 1.625em; + margin-left: 50px; +} +.commentlist .children li.comment .comment-content { + margin: 1.625em 0 0; +} +.comment-meta a { + font-weight: bold; +} +.comment-meta a:focus, +.comment-meta a:active, +.comment-meta a:hover { +} +.commentlist .avatar { + -moz-border-radius: 3px; + border-radius: 3px; + -webkit-box-shadow: 0 1px 2px #ccc; + -moz-box-shadow: 0 1px 2px #ccc; + box-shadow: 0 1px 2px #ccc; + left: -102px; + padding: 0; + position: absolute; + top: 0; +} +.commentlist > li:before { + content: url(images/comment-arrow.png); + left: -21px; + position: absolute; +} +.commentlist > li.pingback:before { + content: ''; +} +.commentlist .children .avatar { + background: none; + -webkit-box-shadow: none; + -moz-box-shadow: none; + box-shadow: none; + left: 2.2em; + padding: 0; + top: 2.2em; +} +a.comment-reply-link { + background: #eee; + -moz-border-radius: 3px; + border-radius: 3px; + color: #666; + display: inline-block; + font-size: 12px; + padding: 0 8px; + text-decoration: none; +} +a.comment-reply-link:hover, +a.comment-reply-link:focus, +a.comment-reply-link:active { + background: #888; + color: #fff; +} +a.comment-reply-link > span { + display: inline-block; + position: relative; + top: -1px; +} + +/* Post author highlighting */ +.commentlist > li.bypostauthor { + background: #ddd; + border-color: #d3d3d3; +} +.commentlist > li.bypostauthor .comment-meta { + color: #575757; +} +.commentlist > li.bypostauthor .comment-meta a:focus, +.commentlist > li.bypostauthor .comment-meta a:active, +.commentlist > li.bypostauthor .comment-meta a:hover { +} +.commentlist > li.bypostauthor:before { + content: url(images/comment-arrow-bypostauthor.png); +} + +/* Post Author threaded comments */ +.commentlist .children > li.bypostauthor { + background: #ddd; + border-color: #d3d3d3; +} + +/* sidebar-page.php comments */ +/* Make sure we have room for our comment avatars */ +.page-template-sidebar-page-php .commentlist > li.comment, +.page-template-sidebar-page-php.commentlist .pingback { + margin-left: 102px; + width: auto; +} +/* And a full-width comment form */ +.page-template-sidebar-page-php #respond { + width: auto; +} + +/* Comment Form */ +#respond { + background: #ddd; + border: 1px solid #d3d3d3; + -moz-border-radius: 3px; + border-radius: 3px; + margin: 0 auto 1.625em; + padding: 1.625em; + position: relative; + width: 68.9%; +} +#respond input[type="text"], +#respond textarea { + background: #fff; + border: 4px solid #eee; + -moz-border-radius: 5px; + border-radius: 5px; + -webkit-box-shadow: inset 0 1px 3px rgba(204,204,204,0.95); + -moz-box-shadow: inset 0 1px 3px rgba(204,204,204,0.95); + box-shadow: inset 0 1px 3px rgba(204,204,204,0.95); + position: relative; + padding: 10px; + text-indent: 80px; +} +#respond .comment-form-author, +#respond .comment-form-email, +#respond .comment-form-url, +#respond .comment-form-comment { + position: relative; +} +#respond .comment-form-author label, +#respond .comment-form-email label, +#respond .comment-form-url label, +#respond .comment-form-comment label { + background: #eee; + -webkit-box-shadow: 1px 2px 2px rgba(204,204,204,0.8); + -moz-box-shadow: 1px 2px 2px rgba(204,204,204,0.8); + box-shadow: 1px 2px 2px rgba(204,204,204,0.8); + color: #555; + display: inline-block; + font-size: 13px; + left: 4px; + min-width: 60px; + padding: 4px 10px; + position: relative; + top: 40px; + z-index: 1; +} +#respond input[type="text"]:focus, +#respond textarea:focus { + text-indent: 0; + z-index: 1; +} +#respond textarea { + resize: vertical; + width: 95%; +} +#respond .comment-form-author .required, +#respond .comment-form-email .required { + color: #bd3500; + font-size: 22px; + font-weight: bold; + left: 75%; + position: absolute; + top: 45px; + z-index: 1; +} +#respond .comment-notes, +#respond .logged-in-as { + font-size: 13px; +} +#respond p { + margin: 10px 0; +} +#respond .form-submit { + float: right; + margin: -20px 0 10px; +} +#respond input#submit { + background: #222; + border: none; + -moz-border-radius: 3px; + border-radius: 3px; + -webkit-box-shadow: 0px 1px 2px rgba(0,0,0,0.3); + -moz-box-shadow: 0px 1px 2px rgba(0,0,0,0.3); + box-shadow: 0px 1px 2px rgba(0,0,0,0.3); + color: #eee; + cursor: pointer; + font-size: 15px; + margin: 20px 0; + padding: 5px 42px 5px 22px; + position: relative; + left: 30px; + text-shadow: 0 -1px 0 rgba(0,0,0,0.3); +} +#respond input#submit:active { + background: #1982d1; + color: #bfddf3; +} +#respond #cancel-comment-reply-link { + color: #666; + margin-left: 10px; + text-decoration: none; +} +#respond .logged-in-as a:hover, +#respond #cancel-comment-reply-link:hover { + text-decoration: underline; +} +.commentlist #respond { + margin: 1.625em 0 0; + width: auto; +} +#reply-title { + color: #373737; + font-size: 24px; + font-weight: bold; + line-height: 30px; +} +#cancel-comment-reply-link { + color: #888; + display: block; + font-size: 10px; + font-weight: normal; + line-height: 2.2em; + letter-spacing: 0.05em; + position: absolute; + right: 1.625em; + text-decoration: none; + text-transform: uppercase; + top: 1.1em; +} +#cancel-comment-reply-link:focus, +#cancel-comment-reply-link:active, +#cancel-comment-reply-link:hover { + color: #ff4b33; +} +#respond label { + line-height: 2.2em; +} +#respond input[type=text] { + display: block; + height: 24px; + width: 75%; +} +#respond p { + font-size: 12px; +} +p.comment-form-comment { + margin: 0; +} +.form-allowed-tags { + display: none; +} + + +/* =Footer +----------------------------------------------- */ + +#colophon { + clear: both; +} +#supplementary { + border-top: 1px solid #ddd; + padding: 1.625em 7.6%; + overflow: hidden; +} + +/* Two Footer Widget Areas */ +#supplementary.two .widget-area { + float: left; + margin-right: 3.7%; + width: 48.1%; +} +#supplementary.two .widget-area + .widget-area { + margin-right: 0; +} + +/* Three Footer Widget Areas */ +#supplementary.three .widget-area { + float: left; + margin-right: 3.7%; + width: 30.85%; +} +#supplementary.three .widget-area + .widget-area + .widget-area { + margin-right: 0; +} + +/* Site Generator Line */ +#site-generator { + background: #f9f9f9; + border-top: 1px solid #ddd; + color: #666; + font-size: 12px; + line-height: 2.2em; + padding: 2.2em 0.5em; + text-align: center; +} +#site-generator a { + color: #555; + font-weight: bold; +} + + +/* =Responsive Structure +----------------------------------------------- */ + +@media (max-width: 800px) { + /* Simplify the basic layout */ + #main #content { + margin: 0 7.6%; + width: auto; + } + #nav-below { + border-bottom: 1px solid #ddd; + margin-bottom: 1.625em; + } + #main #secondary { + float: none; + margin: 0 7.6%; + width: auto; + } + /* Simplify the showcase template */ + .page-template-showcase-php .featured-posts { + min-height: 280px; + } + .featured-posts section.featured-post { + height: auto; + } + .page-template-showcase-php section.recent-posts { + float: none; + margin: 0; + width: 100%; + } + .page-template-showcase-php #main .widget-area { + float: none; + margin: 0; + width: auto; + } + .page-template-showcase-php .other-recent-posts { + border-bottom: 1px solid #ddd; + } + /* Simplify the showcase template when small feature */ + section.featured-post .attachment-small-feature, + .one-column section.featured-post .attachment-small-feature { + border: none; + display: block; + float: left; + height: auto; + margin: 0.625em auto 1.025em; + max-width: 30%; + position: static; + } + article.feature-image.small { + float: right; + margin: 0 0 1.625em; + width: 64%; + } + .one-column article.feature-image.small .entry-summary { + height: auto; + } + article.feature-image.small .entry-summary p a { + left: 0; + padding-left: 20px; + padding-right: 20px; + width: auto; + } + /* Remove the margin on singular articles */ + .singular .entry-header, + .singular .entry-content, + .singular footer.entry-meta, + .singular #comments-title { + width: 100%; + } + /* Simplify the pullquotes and pull styles */ + .singular blockquote.pull { + margin: 0 0 1.625em; + } + .singular .pull.alignleft { + margin: 0 1.625em 0 0; + } + .singular .pull.alignright { + margin: 0 0 0 1.625em; + } + .singular .entry-meta .edit-link a { + left: 0; + position: absolute; + top: 40px; + } + .singular #author-info { + margin: 2.2em -8.8% 0; + padding: 20px 8.8%; + } + /* Make sure we have room for our comment avatars */ + .commentlist { + width: 100%; + } + .commentlist > li.comment, + .commentlist .pingback { + margin-left: 102px; + width: auto; + } + /* And a full-width comment form */ + #respond { + width: auto; + } + /* No need to float footer widgets at this size */ + #colophon #supplementary .widget-area { + float: none; + margin-right: 0; + width: auto; + } + /* No need to float 404 widgets at this size */ + .error404 #main .widget { + float: none; + margin-right: 0; + width: auto; + } + +} +@media (max-width: 650px) { + /* @media (max-width: 650px) Reduce font-sizes for better readability on smaller devices */ + body, input, textarea { + font-size: 13px; + } + #site-title a { + font-size: 24px; + } + #site-description { + font-size: 12px; + } + #access ul { + font-size: 12px; + } + article.intro .entry-content { + font-size: 12px; + } + .entry-title { + font-size: 21px; + } + .featured-post .entry-title { + font-size: 14px; + } + .singular .entry-title { + font-size: 28px; + } + .entry-meta { + font-size: 12px; + } + blockquote { + margin: 0; + } + blockquote.pull { + font-size: 17px; + } + /* Reposition the site title and description slightly */ + #site-title { + padding: 5.30625em 0 0; + } + #site-title, + #site-description { + margin-right: 0; + } + /* Make sure the logo and search form don't collide */ + #branding #searchform { + top: 1.625em !important; + } + /* Floated content doesn't work well at this size */ + .alignleft, + .alignright { + display: block; + float: none; + margin-left: 0; + margin-right: 0; + } + /* Make sure the post-post navigation doesn't collide with anything */ + #nav-single { + display: block; + position: static; + } + .singular .hentry { + padding: 1.625em 0 0; + } + .singular.page .hentry { + padding: 1.625em 0 0; + } + /* Talking avatars take up too much room at this size */ + .commentlist > li.comment, + .commentlist > li.pingback { + margin-left: 0 !important; + } + .commentlist .avatar { + background: transparent; + display: block; + padding: 0; + position: static; + } + .commentlist .children .avatar { + background: none; + left: 2.2em; + padding: 0; + position: absolute; + top: 2.2em; + } + /* Use the available space in the smaller comment form */ + #respond input[type="text"] { + width: 95%; + } + #respond .comment-form-author .required, + #respond .comment-form-email .required { + left: 95%; + } + #content .gallery-columns-3 .gallery-item { + width: 31%; + padding-right: 2%; + } + #content .gallery-columns-3 .gallery-item img { + width: 100%; + height: auto; + } + +} +@media (max-width: 450px) { + #content .gallery-columns-2 .gallery-item { + width: 45%; + padding-right: 4%; + } + #content .gallery-columns-2 .gallery-item img { + width: 100%; + height: auto; + } + +} +@media only screen and (min-device-width: 320px) and (max-device-width: 480px) { + body { + padding: 0; + } + #page { + margin-top: 0; + } + #branding { + border-top: none; + } + +} + + +/* =Print +----------------------------------------------- */ + +@media print { + body { + background: none !important; + font-size: 10pt; + } + footer.entry-meta a[rel=bookmark]:link:after, + footer.entry-meta a[rel=bookmark]:visited:after { + content: " [" attr(href) "] "; /* Show URLs */ + } + #page { + clear: both !important; + display: block !important; + float: none !important; + max-width: 100%; + position: relative !important; + } + #branding { + border-top: none !important; + padding: 0; + } + #branding hgroup { + margin: 0; + } + #site-title a { + font-size: 21pt; + } + #site-description { + font-size: 10pt; + } + #branding #searchform { + display: none; + } + #branding img { + display: none; + } + #access { + display: none; + } + #main { + border-top: none; + box-shadow: none; + } + #primary { + float: left; + margin: 0; + width: 100%; + } + #content { + margin: 0; + width: auto; + } + .singular #content { + margin: 0; + width: 100%; + } + .singular .entry-header .entry-meta { + position: static; + } + .entry-meta .edit-link a { + display: none; + } + #content nav { + display: none; + } + .singular .entry-header, + .singular .entry-content, + .singular footer.entry-meta, + .singular #comments-title { + margin: 0; + width: 100%; + } + .singular .hentry { + padding: 0; + } + .entry-title, + .singular .entry-title { + font-size: 21pt; + } + .entry-meta { + font-size: 10pt; + } + .entry-header .comments-link { + display: none; + } + .page-link { + display: none; + } + .singular #author-info { + background: none; + border-bottom: none; + border-top: none; + margin: 2.2em 0 0; + padding: 0; + } + #respond { + display: none; + } + .widget-area { + display: none; + } + #colophon { + display: none; + } + + /* Comments */ + .commentlist > li.comment { + background: none; + border: 1px solid #ddd; + -moz-border-radius: 3px 3px 3px 3px; + border-radius: 3px 3px 3px 3px; + margin: 0 auto 1.625em; + padding: 1.625em; + position: relative; + width: auto; + } + .commentlist .avatar { + height: 39px; + left: 2.2em; + top: 2.2em; + width: 39px; + } + .commentlist li.comment .comment-meta { + line-height: 1.625em; + margin-left: 50px; + } + .commentlist li.comment .fn { + display: block; + } + .commentlist li.comment .comment-content { + margin: 1.625em 0 0; + } + .commentlist .comment-edit-link { + display: none; + } + .commentlist > li::before, + .commentlist > li.bypostauthor::before { + content: ''; + } + .commentlist .reply { + display: none; + } + + /* Post author highlighting */ + .commentlist > li.bypostauthor { + color: #444; + } + .commentlist > li.bypostauthor .comment-meta { + color: #666; + } + .commentlist > li.bypostauthor:before { + content: none; + } + + /* Post Author threaded comments */ + .commentlist .children > li.bypostauthor { + background: #fff; + border-color: #ddd; + } + .commentlist .children > li.bypostauthor > article, + .commentlist .children > li.bypostauthor > article .comment-meta { + color: #666; + } + +} + + +/* =IE7 +----------------------------------------------- */ + +#ie7 article.intro { + margin-left: -7.6%; + margin-right: -7.6%; + padding-left: -7.6%; + padding-right: -7.6%; + max-width: 1000px; +} +#ie7 section.featured-post { + margin-left: -7.6%; + margin-right: -7.6%; + max-width: 850px; +} +#ie7 section.recent-posts { + margin-right: 7.6%; +} + +/* =IE8 +----------------------------------------------- */ + +#ie8 section.feature-image.large img { + width: 100%; +} +/** + * Theme Name: Spark + * Description: Child theme for the twentyeleven theme + * Author: Andy Konwinski + * Template: twentyeleven + **/ + +@import url("../twentyeleven/style.css"); + +ul { + margin-bottom: 10px; +} + +.shadow { + -moz-box-shadow: -1px 1px 5px 1px #ccc; + -webkit-box-shadow: -1px 1px 5px 1px #ccc; + box-shadow: -1px 1px 5px 1px #ccc; +} + +/* Make embedded video containers look better with responsive design */ +.video-container { + position: relative; + padding-bottom: 90%; + padding-top: 25px; + height: 0; + overflow: hidden; + margin-bottom: 20px; +} + +.video-container iframe, +.video-container object, +.video-container embed { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; +} + +.entry-content { + padding: 0 !important; +} + +.singular.page .hentry, .singular .hentry { + padding: 0; +} + +.hentry { + border-bottom: none; +} + +.singular .entry-header { + width: 100%; +} + +.singular .entry-content { + width: 100%; + margin-bottom: 30px; +} + +.single-post .entry-content { + /*border-bottom: thin lightgray dotted;*/ +} + +.singular #content { + width: 100%; + margin: 0; +} + +.archive #content { + width: 100%; +} + +.page-title { + margin-bottom: 0; +} + +.entry-header { + margin-bottom: 5px; +} + +.entry-header .comments-link a { + top: 0; +} + +/* =Footer +----------------------------------------------- */ +/* Site Generator Line */ +#site-generator { + background: white; + border-top: 1px solid #ddd; + color: #666; + font-size: 12px; + line-height: 1.5em; + padding: 1.5em 0.5em; + text-align: center; + height: 39px; +} + +#site-generator p { + display: inline-block; + padding-top: 15px; + color: #777; +} + +#site-generator a { + text-decoration: underline; + font-weight: normal; +} + +#site-generator .amp-logo { + display: inline-block; + text-decoration: none; + width: 167px; + height: 39px; + float: right; +} + +.error404 .widgettitle { + line-height: 1.5em; + text-transform: none; + text-align: left; + font-size: 17px; +} + +/* =Menu +-------------------------------------------------------------- */ +#access { + background: #222; /* Show a solid color for older browsers */ + background: -moz-linear-gradient(#252525, #0a0a0a); + background: -o-linear-gradient(#252525, #0a0a0a); + background: -webkit-gradient(linear, 0% 0%, 0% 100%, from(#252525), to(#0a0a0a)); /* older webkit syntax */ + background: -webkit-linear-gradient(#484A63, #0a0a0a); + -webkit-box-shadow: rgba(0, 0, 0, 0.4) 0px 1px 2px; + -moz-box-shadow: rgba(0, 0, 0, 0.4) 0px 1px 2px; + box-shadow: rgba(0, 0, 0, 0.4) 0px 1px 2px; + clear: both; + display: block; + float: left; + margin: 0 auto; + width: 100%; + text-align: center; +} + +/* =Structure +----------------------------------------------- */ +.singular .entry-meta .edit-link a { + position: absolute; + left: 0; + top: -18px; +} + +.singular #comments-title { + margin: 0; +} +.singular footer.entry-meta { + margin: 0; +} +#comments-title { + text-transform: none; + line-height: 1.5em; + font-size: 1.5em !important; +} + +/* Copied in from spark-project.org stylesheet */ +/*---------------------------------------------*/ + +*{margin:0; padding:0;} +body { + font-family: Verdana, Arial, "Trebuchet MS", Sans-Serif, Georgia, Courier, "Times New Roman", Serif; + font-size: 12px; + background: #fff; + color: #333; + margin:0; + padding:0; +} + +body.custom-background { + background-color: white !important; +} + +a { color: #06f; text-decoration: none; } +a:hover { text-decoration: none; color : #000; } + +img { border: none; padding: 6px; } +img a {border:none;} + +#page { + max-width: 795px; + margin: 0 auto; +} + +#site-title a { + color: #000; + padding: 0; + line-height: 75px; + font-size: 70px; + font-weight: bold; + font-family: Corbel, "Trebuchet MS", Verdana, Arial, Sans-Serif; +} +#site-title a:hover { + color: #000; +} +#site-description { + color: #06f; + font-size: 17px; + font-weight: 100; + margin-bottom: 2.7em; +} +/* +#header h2 a { +#header h2 a:hover { color: #06f; } +*/ + +#menu { + background: #454645 url(images/menu.jpg); + height: 36px; + line-height: 36px; +} + +#menu ul { + list-style-type: none; + margin: 0; + padding-left: 10px; +} +#menu ul li { + display: block; + float: left; +} +#menu ul li a { + display: block; + padding: 0 10px 0 10px; + text-decoration: none; + color: #fff; + font-size: 13px; + font-weight: 600; +} +#menu ul li a:hover { + color: #fff; + background: url(images/menuover.png) repeat-x; +} + +#wrapper { + padding: 0 0 0 20px; +} + +#primary { + width: 100%; +} + +#main { + width: 95%; + margin: 0 2.5%; +} +#content { + float: left; + margin: 0; + text-align: justify; + width: 58%; +} + +#content nav{ + padding-bottom: 0; +} + +#footer { + background: #fff; + margin: 0 auto; + font-weight: 100; + font-size: 12px; + text-align: center; + color #999; + height: 50px; + line-height: 50px; +} +#footer p { color: #999; } +#footer a { color: #999; text-decoration: none; } +#footer a:hover { text-decoration: underline; } + + +#content h1 { + font-weight: 100; + font-size: 125%; + margin: 0 0 4px; padding: 0 0 3px; + color: #888; +} + +#content h2 { + font-weight: 100; + font-size: 23px; + margin: 0 0 4px; padding: 0 0 3px; + color: #888; +} + +#content h3 { + font-weight: bold; + font-size: 110%; + margin-top: 15px; + margin-bottom: 4px; + color: #333; +} + +#content ul, #content ol { + padding: 0 30px; +} + +#content ul li { + list-style-type: square; +} + +#secondary { + width: 35%; + padding-left: 20px; + margin-right: 10px; +} + +@media only all and (min-width: 800px) { + #secondary { + border-left: 1px solid #ddd; + } +} + +#secondary img { padding: 0; margin: 0; } + +#secondary ul { + list-style-type: none; +} + +#secondary h2 { + height: 50px; + line-height: 50px; + font-weight: 600; + font-size: 14px; + margin: 30px 0 0 0; + color: #111; +} + +.code { + font-family: "Monaco", "Courier New", Monospace; + border-top: 1px solid #aaa; + border-bottom: 1px solid #aaa; + padding-top: 8px; + padding-bottom: 8px; + margin-bottom: 5px; + line-height: 140%; +} + +.keyword { color: #888; font-weight: bold; } +.sparkop { color: #06f; } +.closure { color: #f09; } +.closure .keyword { color: #f09; font-weight: normal; } +.string { color: #393; } + +.caption { width: 100%; text-align: center; font-weight: bold; } + +.latestnewswidget { + margin-bottom: 0; +} + +.latestnewswidget div { + width: 95%; + background: #ffefaa; + padding: 4px 10px 4px 10px; + margin-bottom: 15px; + border-bottom: none; +} + +.widget a { + font-weight: normal; +} + +#content p { margin-bottom: 10px; } + +.question { font-weight: bold; font-size: 110%; margin-top: 15px; } + +code, tt { font-family: "Monaco", "Courier New", Monospace; font-size: 12px; } + +.post-info { font-size: 10px; color: #444; } + +#branding { + border-top: none; + padding: 0px; +} +#branding img { border: none; padding: 0px 0px 0px 6px; } + +#comments { + font-size: 12px; +} + +#respond, .commentlist > li.bypostauthor { + background: #ddd; +} + +#respond .comment-notes { + font-size: 12px; +} + +#respond .logged-in-as { + font-size: 12px; +} + +#respond p, #respond h3 { + margin: 0px; +} + +#respond .comment-form-author label, +#respond .comment-form-email label, +#respond .comment-form-url label, +#respond .comment-form-comment label { + font-size: 12px; + top: 40px; + padding-top: 8px; +} diff --git a/documentation.md b/documentation.md new file mode 100644 index 000000000..ef488ebdd --- /dev/null +++ b/documentation.md @@ -0,0 +1,84 @@ +--- +layout: global +title: Documentation +type: "page singular" +navigation: + weight: 3 + show: true +--- + +<h2>Spark Documentation</h2> + +<p>Setup instructions, programming guides, and other documentation are available for each version of Spark below:</p> + +<ul> + <li><a href="{{site.url}}docs/latest/">Spark 0.7.3 (latest release)</a></li> + <li><a href="{{site.url}}docs/0.6.2/">Spark 0.6.2</a></li> + <li><a href="https://github.com/mesos/spark/wiki/Spark-0.5-Documentation">Spark 0.5.x</a> (hosted on GitHub)</li> +</ul> + +<p>Read these documents to get started with Spark. In addition, this page lists a few other external resources for learning Spark.</p> + +<h3>Video Tutorials</h3> + +<ul> + <li><a href="{{site.url}}screencasts/1-first-steps-with-spark.html">Screencast 1: First Steps with Spark</a></li> + <li><a href="{{site.url}}screencasts/2-spark-documentation-overview.html">Screencast 2: Spark Documentation Overview</a></li> +</ul> + +<h3>Hands-On Exercises</h3> + +<ul> + <li><a href="http://ampcamp.berkeley.edu/big-data-mini-course/">Hands-on exercises from Strata 2013</a> are available online. These exercises let you launch a small EC2 cluster, load a dataset, and query it with Spark, Shark, and Spark Streaming.</li> +</ul> + +<h3>AMP Camp Slides and Videos</h3> + +<ul> + <li>The AMP Lab hosted a <a href="http://ampcamp.berkeley.edu/agenda-2012/">two-day training camp</a> on Spark, Shark and "big data" analysis in August 2012. <a href="http://ampcamp.berkeley.edu/agenda-2012/">Slides and videos</a> are available online for free. + </li> +</ul> + +<h3>Presentations</h3> + +<ul> + <li> + <a href="{{site.url}}talks/overview.pptx">Overview of Spark</a> (<a href="{{site.url}}talks/overview.pptx">pptx</a>) (<a href="/talks/overview.pdf">pdf</a>) + </li> + <li>Programming Spark tutorial at AMP Camp: <a href="http://ampcamp.berkeley.edu/wp-content/uploads/2012/06/matei-zaharia-part-1-amp-camp-2012-spark-intro.pdf">part 1 (pdf)</a>, <a href="http://ampcamp.berkeley.edu/wp-content/uploads/2012/06/matei-zaharia-part-2-amp-camp-2012-standalone-programs.pdf">part 2 (pdf)</a>, <a href="http://www.youtube.com/watch?v=7k4yDKBYOcw">video</a>. + </li> + <li><a href="{{site.url}}talks/strata_spark_streaming.ppt">Overview of Spark Streaming</a> (<a href="{{site.url}}talks/strata_spark_streaming.ppt">ppt</a>) (<a href="{{site.url}}talks/strata_spark_streaming.pdf">pdf</a>) +</ul> + +<h3>External Tutorials, Development Blogs, and Talks</h3> + +<ul> + <li><a href="http://www.slideshare.net/EvanChan2/cassandra2013-spark-talk-final">Real-time Analytics with Cassandra, Spark, and Shark</a> -- presentation by Evan Chan from Ooyala at the 2013 Cassandra Summit</li> + <li><a href="http://syndeticlogic.net/?p=311">Getting Spark Setup in Eclipse</a> -- Developer blog post by Spark community member James Percent</li> + <li><a href="http://aws.amazon.com/articles/Elastic-MapReduce/4926593393724923">Run Spark and Shark on Amazon Elastic MapReduce</a> -- Article by Amazon AWS Elastic MapReduce team member Parviz Deyhim</li> + <li><a href="http://blog.quantifind.com/posts/spark-unit-test/">Unit testing with Spark</a> -- Quantifind tech blog post by Imran Rashid</li> + <li><a href="http://blog.quantifind.com/posts/logging-post/">Configuring Spark logs</a> -- Quantifind tech blog by Imran Rashid</li> + <li><a href="http://www.ibm.com/developerworks/library/os-spark/">Spark, an alternative for fast data analytics</a> -- IBM Developer Works article by M. Tim Jones</li> +</ul> + +<h3>Research Papers</h3> + +<ul> + <li> + <a href="http://www.eecs.berkeley.edu/Pubs/TechRpts/2012/EECS-2012-214.pdf">Shark: SQL and Rich Analytics at Scale</a>. Reynold Xin, Joshua Rosen, Matei Zaharia, Michael J. Franklin, Scott Shenker, Ion Stoica. <em>Technical Report UCB/EECS-2012-214</em>. November 2012. + </li> + <li> + <a href="http://www.cs.berkeley.edu/~matei/papers/2012/hotcloud_spark_streaming.pdf">Discretized Streams: An Efficient and Fault-Tolerant Model for Stream Processing on Large Clusters</a>. Matei Zaharia, Tathagata Das, Haoyuan Li, Scott Shenker, Ion Stoica. <em>HotCloud 2012</em>. June 2012. + </li> + <li> + <a href="http://www.cs.berkeley.edu/~matei/papers/2012/sigmod_shark_demo.pdf">Shark: Fast Data Analysis Using Coarse-grained Distributed Memory</a> (demo). Cliff Engle, Antonio Lupher, Reynold Xin, Matei Zaharia, Haoyuan Li, Scott Shenker, Ion Stoica. <em>SIGMOD 2012</em>. May 2012. <b>Best Demo Award</b>. + </li> + <li> + <a href="http://www.cs.berkeley.edu/~matei/papers/2012/nsdi_spark.pdf">Resilient Distributed Datasets: A Fault-Tolerant Abstraction for In-Memory Cluster Computing</a>. Matei Zaharia, Mosharaf Chowdhury, Tathagata Das, Ankur Dave, Justin Ma, Murphy McCauley, Michael J. Franklin, Scott Shenker, Ion Stoica. <em>NSDI 2012</em>. April 2012. <b>Best Paper Award</b> and <b>Honorable Mention for Community Award</b>. + </li> + <li> + <a href="http://www.cs.berkeley.edu/~matei/papers/2011/tr_spark.pdf">Resilient Distributed Datasets: A Fault-Tolerant Abstraction for In-Memory Cluster Computing</a>. Matei Zaharia, Mosharaf Chowdhury, Tathagata Das, Ankur Dave, Justin Ma, Murphy McCauley, Michael J. Franklin, Scott Shenker, Ion Stoica. <em>Technical Report UCB/EECS-2011-82</em>. July 2011.</li> + <li> + <a href="http://www.cs.berkeley.edu/~matei/papers/2010/hotcloud_spark.pdf">Spark: Cluster Computing with Working Sets</a>. Matei Zaharia, Mosharaf Chowdhury, Michael J. Franklin, Scott Shenker, Ion Stoica. <em>HotCloud 2010</em>. June 2010. + </li> +</ul>
\ No newline at end of file diff --git a/downloads.md b/downloads.md new file mode 100644 index 000000000..65dbdf079 --- /dev/null +++ b/downloads.md @@ -0,0 +1,31 @@ +--- +layout: global +title: Downloads +type: singular +navigation: + weight: 3 + show: true +--- + +<h2>Download Spark</h2> +The latest release of Spark is 0.7.3. You can either download it as a <a href="/download/spark-0.7.3-sources.tgz">source package</a> (4 MB tar.gz) or get prebuilt packages for <a href="/download/spark-0.7.3-prebuilt-hadoop1.tgz">Hadoop 1 / CDH3</a> or <a href="/download/spark-0.7.3-prebuilt-cdh4.tgz">CDH 4</a> (61 MB tar.gz). + +If you are interested in working with the newest under-development code or contributing to Spark development, you can also check out the master branch from Git: <tt>git clone git://github.com/mesos/spark.git</tt>. + +Once you've downloaded Spark, you can find instructions for installing and building it on the <a href="{{site.url}}documentation.html">documentation page</a>. +<h3>Previous Releases</h3> +<ul> + <li><a href="http://spark-project.org/download/spark-0.7.3-sources.tgz">Spark 0.7.3</a> (July 16, 2013) <a href="{{site.url}}releases/spark-release-0-7-3.html">(release notes)</a> (prebuilt: <a href="/download/spark-0.7.3-prebuilt-hadoop1.tgz">Hadoop 1 / CDH3</a>, <a href="{{site.url}}download/spark-0.7.3-prebuilt-cdh4.tgz">CDH 4</a>)</li> + <li><a href="http://spark-project.org/download/spark-0.7.2-sources.tgz">Spark 0.7.2</a> (June 2, 2013) <a href="{{site.url}}releases/spark-release-0-7-2.html">(release notes)</a> (prebuilt: <a href="/download/spark-0.7.2-prebuilt-hadoop1.tgz">Hadoop 1 / CDH3</a>, <a href="/download/spark-0.7.2-prebuilt-cdh4.tgz">CDH 4</a>)</li> + <li><a href="http://spark-project.org/download/spark-0.7.0-sources.tgz">Spark 0.7.0</a> (February 27, 2013) <a href="{{site.url}}releases/spark-release-0-7-0.html">(release notes)</a></li> + <li><a href="http://spark-project.org/download/spark-0.6.2-sources.tgz">Spark 0.6.2</a> (February 7, 2013) <a href="{{site.url}}releases/spark-release-0-6-2.html">(release notes)</a></li> + <li><a href="http://spark-project.org/download-spark-0.6.1-sources-tgz">Spark 0.6.1</a> (November 22, 2012) <a href="{{site.url}}releases/spark-release-0-6-1.html">(release notes)</a></li> + <li><a href="http://spark-project.org/download-spark-0.5.2-sources-tgz">Spark 0.5.2</a> (November 22, 2012) <a href="{{site.url}}releases/spark-release-0-5-2.html">(release notes)</a></li> + <li><a href="http://spark-project.org/download-spark-0.6.0-sources-tgz">Spark 0.6.0</a> (October 15, 2012) <a href="{{site.url}}releases/spark-release-0-6-0.html">(release notes)</a></li> + <li><a href="http://spark-project.org/download-spark-0.5.1-sources-tgz">Spark 0.5.1</a> (October 11, 2012) <a href="{{site.url}}releases/spark-release-0-5-1.html">(release notes)</a></li> + <li><a href="http://spark-project.org/download-spark-0.5.0-sources-tgz">Spark 0.5.0</a> (June 12, 2012) <a href="{{site.url}}releases/spark-release-0-5-0.html">(release notes)</a></li> + <li><a href="http://spark-project.org/download-spark-0.3-for-scala-2-9-sources-tgz">Spark 0.3 for Scala 2.9</a> (July 14, 2011) <a href="{{site.url}}releases/spark-release-0-3.html">(release notes)</a></li> + <li><a href="http://spark-project.org/download-spark-0.3-for-scala-2-8-sources-tgz">Spark 0.3 for Scala 2.8</a> (July 14, 2011) <a href="{{site.url}}releases/spark-release-0-3.html">(release notes)</a></li> + <li><a href="http://spark-project.org/download-spark-0.2-sources-tgz">Spark 0.2</a> (May 24, 2011)</li> +</ul> + diff --git a/examples.md b/examples.md new file mode 100644 index 000000000..36e7b9658 --- /dev/null +++ b/examples.md @@ -0,0 +1,99 @@ +--- +layout: global +title: Examples +type: "page singular" +navigation: + weight: 4 + show: true +--- +<h2>Spark Examples</h2> + +Spark is built around <em>distributed datasets</em> that support types of parallel operations: transformations, which are lazy and yield another distributed dataset (e.g., <code>map</code>, <code>filter</code>, and <code>join</code>), and actions, which force the computation of a dataset and return a result (e.g., <code>count</code>). The following examples show off some of the available operations and features. + +<h3>Text Search</h3> + +In this example, we search through the error messages in a log file: + +<p> +</p><div class="code"> +<span class="keyword">val</span> file = spark.textFile(<span class="string">"hdfs://..."</span>)<br> +<span class="keyword">val</span> errors = file.<span class="sparkop">filter</span>(<span class="closure">line => line.contains("ERROR")</span>)<br> +<span class="comment">// Count all the errors</span><br> +errors.<span class="sparkop">count</span>()<br> +<span class="comment">// Count errors mentioning MySQL</span><br> +errors.<span class="sparkop">filter</span>(<span class="closure">line => line.contains("MySQL")</span>).<span class="sparkop">count</span>()<br> +<span class="comment">// Fetch the MySQL errors as an array of strings</span><br> +errors.<span class="sparkop">filter</span>(<span class="closure">line => line.contains("MySQL")</span>).<span class="sparkop">collect</span>()<br> +</div> +<p></p> + +<p>The red code fragments are Scala function literals (closures) that get passed automatically to the cluster. The blue ones are Spark operations.</p> + +<h3>In-Memory Text Search</h3> + +<p>Spark can <em>cache</em> datasets in memory to speed up reuse. In the example above, we can load just the error messages in RAM using:</p> + +<p> +</p><div class="code"> +errors.<span class="sparkop">cache</span>() +</div> +<p></p> + +<p>After the first action that uses <code>errors</code>, later ones will be much faster.</p> + + +<h3>Word Count</h3> + +<p>In this example, we use a few more transformations to build a dataset of (String, Int) pairs called <code>counts</code> and then save it to a file.</p> + +<p> +</p><div class="code"> +<span class="keyword">val</span> file = spark.textFile(<span class="string">"hdfs://..."</span>)<br> +<span class="keyword">val</span> counts = file.<span class="sparkop">flatMap</span>(<span class="closure">line => line.split(" ")</span>)<br> + .<span class="sparkop">map</span>(<span class="closure">word => (word, 1)</span>)<br> + .<span class="sparkop">reduceByKey</span>(<span class="closure">_ + _</span>)<br> +counts.<span class="sparkop">saveAsTextFile</span>(<span class="string">"hdfs://..."</span>) +</div> +<p></p> + +<h3>Estimating Pi</h3> + +<p>Spark can also be used for compute-intensive tasks. This code estimates <span style="font-family: serif; font-size: 120%;">π</span> by "throwing darts" at a circle. We pick random points in the unit square ((0, 0) to (1,1)) and see how many fall in the unit circle. The fraction should be <span style="font-family: serif; font-size: 120%;">π / 4</span>, so we use this to get our estimate.</p> + +<p> +</p><div class="code"> +<span class="keyword">val</span> count = spark.parallelize(1 to NUM_SAMPLES).<span class="sparkop">map</span>(<span class="closure">i =><br> + <span class="keyword">val</span> x = Math.random<br> + <span class="keyword">val</span> y = Math.random<br> + <span class="keyword">if</span> (x*x + y*y < 1) 1.0 <span class="keyword">else</span> 0.0<br> +</span>).<span class="sparkop">reduce</span>(<span class="closure">_ + _</span>)<br> +println(<span class="string">"Pi is roughly "</span> + 4 * count / NUM_SAMPLES)<br> +</div> +<p></p> + +<h3>Logistic Regression</h3> + +<p>This is an iterative machine learning algorithm that seeks to find the best hyperplane that separates two sets of points in a multi-dimensional feature space. It can be used to classify messages into spam vs non-spam, for example. Because the algorithm applies the same MapReduce operation repeatedly to the same dataset, it benefits greatly from caching the input data in RAM across iterations.</p> + +<p> +</p><div class="code"> +<span class="keyword">val</span> points = spark.textFile(...).<span class="sparkop">map</span>(parsePoint).<span class="sparkop">cache</span>()<br> +<span class="keyword">var</span> w = Vector.random(D) <span class="comment">// current separating plane</span><br> +<span class="keyword">for</span> (i <- 1 to ITERATIONS) {<br> + <span class="keyword">val</span> gradient = points.<span class="sparkop">map</span>(<span class="closure">p =><br> + (1 / (1 + exp(-p.y*(w dot p.x))) - 1) * p.y * p.x<br> + </span>).<span class="sparkop">reduce</span>(<span class="closure">_ + _</span>)<br> + w -= gradient<br> +}<br> +println(<span class="string">"Final separating plane: "</span> + w)<br> +</div> +<p></p> + +<p>Note that <code>w</code> gets shipped automatically to the cluster with every <code>map</code> call.</p> + +<p>The graph below compares the performance of this Spark program against a Hadoop implementation on 30 GB of data on an 80-core cluster, showing the benefit of in-memory caching:</p> + +<p align="center"> +<img src="{{site.url}}images/spark-lr.png" alt="Logistic regression performance in Spark vs Hadoop"> +</p> + @@ -0,0 +1,39 @@ +--- +layout: global +title: FAQ +type: "page singular" +navigation: + weight: 7 + show: true +--- +<h2>Spark FAQ</h2> + +<p class="question">Is Spark a modified version of Hadoop?</p> +<p class="answer">No. Spark is a completely separate codebase optimized for low latency, although it can load data from any Hadoop input source (InputFormat).</p> + +<p class="question">Which languages does Spark support?</p> +<p class="answer">Starting in version 0.7, Spark supports Scala, Java and Python.</p> + +<p class="question">Does Spark require modified versions of Scala or Python?</p> +<p class="answer">No. Spark requires no changes to Scala or compiler plugins. The Python API uses the standard CPython implementation, and can call into existing C libraries for Python such as NumPy.</p> + +<p class="question">What happens when a cached dataset does not fit in memory?</p> +<p class="answer">Spark can either spill it to disk or recompute the partitions that don't fit in RAM each time they are requested. By default, it uses recomputation, but you can set a dataset's <a href="{{site.url}}docs/latest/scala-programming-guide.html#rdd-persistence">storage level</a> to <tt>MEMORY_AND_DISK</tt> to avoid this. </p> + +<p class="question">How can I run Spark on a cluster?</p> +<p class="answer">You can use either the <a href="{{site.url}}docs/latest/spark-standalone.html">standalone deploy mode</a>, which depends only on Java, or the <a href="{{site.url}}docs/latest/running-on-mesos.html">Apache Mesos</a> cluster manager.</p> +<p>Note that you can also run Spark locally (possibly on multiple cores) without any special setup by just passing <tt>local[N]</tt> as the master URL, where <tt>N</tt> is the number of parallel threads you want.</p> + +<p class="question">I don't know Scala; how hard is it to pick it up to use Spark?</p> +<p class="answer">Scala itself is pretty easy to pick up if you have Java experience. Check out <a href="http://www.artima.com/scalazine/articles/steps.html">First Steps to Scala</a> for a quick introduction, the <a href="http://www.scala-lang.org/docu/files/ScalaTutorial.pdf">Scala tutorial for Java programmers</a>, or the free online book <a href="http://www.artima.com/pins1ed/">Programming in Scala</a>.</p> +<p>Spark 0.6 also added a <a href="{{site.url}}docs/latest/java-programming-guide.html">Java API</a>, letting you use Spark from Java, and Spark 0.7 added a <a href="{{site.url}}docs/latest/python-programming-guide.html">Python API</a>.</p> + +<p class="question">What license is Spark under?</p> +<p class="answer">Spark is open source under the liberal <a href="https://github.com/mesos/spark/blob/master/LICENSE">BSD license</a>.</p> + +<p class="question">How can I contribute to Spark?</p> +<p class="answer">Contact the <a href="http://groups.google.com/group/spark-users">mailing list</a> or send us a pull request on GitHub. We're glad to hear about your experience using Spark and to accept patches </p> +<p>If you would like to report an issue, post it to the <a href="https://spark-project.atlassian.net/browse/SPARK">Spark issue tracker</a>.</p> + +<p class="question">Where can I get more help?</p> +<p class="answer">Please post on the <a href="http://groups.google.com/group/spark-users">spark-users</a> mailing list. We'll be glad to help!</p>
\ No newline at end of file diff --git a/favicon.ico b/favicon.ico Binary files differnew file mode 100644 index 000000000..6944559c8 --- /dev/null +++ b/favicon.ico diff --git a/images/amplab-small.png b/images/amplab-small.png Binary files differnew file mode 100644 index 000000000..4547fd094 --- /dev/null +++ b/images/amplab-small.png diff --git a/images/download.png b/images/download.png Binary files differnew file mode 100644 index 000000000..ce1c94b1d --- /dev/null +++ b/images/download.png diff --git a/images/spark-lr.png b/images/spark-lr.png Binary files differnew file mode 100644 index 000000000..a9729b519 --- /dev/null +++ b/images/spark-lr.png diff --git a/images/spark-project-header1.png b/images/spark-project-header1.png Binary files differnew file mode 100644 index 000000000..913243d98 --- /dev/null +++ b/images/spark-project-header1.png diff --git a/index.md b/index.md new file mode 100644 index 000000000..223819f94 --- /dev/null +++ b/index.md @@ -0,0 +1,52 @@ +--- +layout: global +title: Home +type: page +navigation: + weight: 1 + show: true +--- +## What is Spark? +Spark is an open source cluster computing system that aims to make data analytics <em>fast</em> — both fast to run and fast to write. +To run programs faster, Spark provides primitives for in-memory cluster computing: your job can load data into memory and query it repeatedly much more quickly than with disk-based systems like Hadoop MapReduce. +To make programming faster, Spark provides clean, concise APIs in <a href="http://www.scala-lang.org" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://www.scala-lang.org']);">Scala</a>, <a href="{{site.url}}docs/latest/quick-start.html#a-standalone-job-in-java" >Java</a> and <a href="{{site.url}}docs/latest/quick-start.html#a-standalone-job-in-python" >Python</a>. You can also use Spark interactively from the Scala and Python shells to rapidly query big datasets. + +## What can it do? +Spark was initially developed for two applications where keeping data in memory helps: <em>iterative</em> algorithms, which are common in machine learning, and <em>interactive</em> data mining. In both cases, Spark can run up to <b>100x</b> faster than Hadoop MapReduce. However, you can use Spark for general data processing too. Check out our <a href="{{site.url}}examples.html" >example jobs</a>. +Spark is also the engine behind <a href="http://shark.cs.berkeley.edu" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://shark.cs.berkeley.edu']);">Shark</a>, a fully <a href="http://hive.apache.org" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://hive.apache.org']);">Apache Hive</a>-compatible data warehousing system that can run 100x faster than Hive. +While Spark is a new engine, it can access any data source supported by Hadoop, making it easy to run over existing data. + +## Who uses it? +Spark was developed in the <a href="https://amplab.cs.berkeley.edu" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://amplab.cs.berkeley.edu']);">UC Berkeley AMPLab</a>. It’s used by several groups of researchers at Berkeley to run large-scale applications such as spam filtering and traffic prediction. It’s also used to accelerate data analytics at <a href="http://www.yahoo.com" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://www.yahoo.com']);">Yahoo!</a>, <a href="http://www.conviva.com" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://www.conviva.com']);">Conviva</a>, <a href="http://www.quantifind.com" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://www.quantifind.com']);">Quantifind</a>, and other companies — in total, 17 companies have contributed to Spark! Spark is <a href="https://github.com/mesos/spark" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://github.com']);">open source</a> under a BSD license, so <a href="{{site.url}}downloads.html" >download</a> it to check it out. + +## Apache Incubator notice +Apache Spark is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF. + +{% sidebar %} + <h3 class="widget-title">Latest News</h3> + <div class="latestnewswidget"> + {% for post in site.categories.news limit:4 %} + <div><a href="{{ post.url }}">{{ post.title }}</a> <span class="post-info">({{post.date| date:"%B %d, %Y"}})</span></div> + {% endfor %} + </div> + + <div style="text-align:right"><a href="{{site.url}}news/index.html">News Archive</a></div> + + <!-- Not porting the following to Pygments since it becomes a lot less colorful --> + + <div class="code" style="margin-top: 20px;"> + file = spark.textFile(<span class="string">"hdfs://..."</span>)<br> + <br> + file.<span class="sparkop">flatMap</span>(<span class="closure">line => line.split(" ")</span>)<br> + .<span class="sparkop">map</span>(<span class="closure">word => (word, 1)</span>)<br> + .<span class="sparkop">reduceByKey</span>(<span class="closure">_ + _</span>) + </div> + <div class="caption">Word Count implemented in Spark</div> + + <div align="center" style="margin-top: 20px;"> + <img src="{{site.url}}images/spark-lr.png" alt="Logistic regression performance in Spark vs Hadoop"> + </div> + <div class="caption">Logistic regression in Spark vs Hadoop</div> + <h2 style="text-align:center"><a href="{{site.url}}downloads"><img src="{{site.url}}images/download.png" alt="Download" style="vertical-align: middle"> Download Spark</a></h2> +{% endsidebar %} + diff --git a/mailing-lists.md b/mailing-lists.md new file mode 100644 index 000000000..60c5f0f88 --- /dev/null +++ b/mailing-lists.md @@ -0,0 +1,39 @@ +--- +layout: global +title: Mailing Lists +type: "page singular" +navigation: + weight: 5 + show: true +--- +<h2>Spark Mailing Lists</h2> + +<p>Get help using Spark or contribute to the project on our mailing lists:</p> +<ul> + <li> + <a href="http://mail-archives.apache.org/mod_mbox/incubator-spark-user/">user@spark.incubator.apache.org</a> is for usage questions, help, and announcements. <a href="mailto:user-subscribe@spark.incubator.apache.org?subject=(send%20this%20email%20to%20subscribe)">(subscribe)</a> <a href="http://mail-archives.apache.org/mod_mbox/incubator-spark-user/">(archives)</a> + </li> + <li> + <a href="http://mail-archives.apache.org/mod_mbox/incubator-spark-dev/">dev@spark.incubator.apache.org</a> is for people who want to contribute code to Spark. <a href="mailto:dev-subscribe@spark.incubator.apache.org?subject=(send%20this%20email%20to%20subscribe)">(subscribe)</a> <a href="http://mail-archives.apache.org/mod_mbox/incubator-spark-dev/">(archives)</a> + </li> +</ul> + +<h2>User Meetups</h2> + +<p>We host a regular <a href="http://www.meetup.com/spark-users/">Bay Area Spark User Meetup</a> where you can meet the community in person. The meetup.com page also contains an <a href="http://www.meetup.com/spark-users/events/past/">archive of past meetups</a>, and <a href="http://www.meetup.com/spark-users/files/">slides</a> from them.</p> + +<h2>Issue Tracker</h2> + +<p>The project tracks bugs and new features on <a href="https://spark-project.atlassian.net/browse/SPARK">JIRA</a>. If you'd like, you can also subscribe to the <a href="http://groups.google.com/group/spark-issues">spark-issues</a> group to receive emails about new or updated issues.</p> + +<h2>Old Mailing Lists</h2> + +<p>The following Google Groups contain Spark usage and developer discussions from before the transition to Apache. They will become read-only in September 2013:</p> +<ul> + <li> + <a href="http://groups.google.com/group/spark-users">spark-users</a> -- usage questions, help, and announcements. + </li> + <li> + <a href="http://groups.google.com/group/spark-developers">spark-developers</a> -- code contributor discussion. + </li> +</ul> diff --git a/news/_posts/2012-01-10-spark-meetups.html b/news/_posts/2012-01-10-spark-meetups.html new file mode 100644 index 000000000..5c5ba0faa --- /dev/null +++ b/news/_posts/2012-01-10-spark-meetups.html @@ -0,0 +1,11 @@ +--- +layout: post +title: We've started hosting a Bay Area Spark User Meetup +categories: +- News +tags: [] +status: publish +type: post +published: true +--- +We've started hosting a regular <a href="http://www.meetup.com/spark-users/">Bay Area Spark User Meetup</a>. Sign up on the meetup.com page to be notified about events and meet other Spark developers and users. diff --git a/news/_posts/2012-04-25-nsdi-paper.html b/news/_posts/2012-04-25-nsdi-paper.html new file mode 100644 index 000000000..4f01d52ab --- /dev/null +++ b/news/_posts/2012-04-25-nsdi-paper.html @@ -0,0 +1,14 @@ +--- +layout: post +title: Spark wins Best Paper Award at USENIX NSDI +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '1' + _wp_old_slug: hello-world +--- +Our <a href="http://www.cs.berkeley.edu/~matei/papers/2012/nsdi_spark.pdf">paper on Spark</a> won the Best Paper Award at the <a href="http://www.usenix.org/nsdi12/">USENIX NSDI conference</a>. You can see a video of the talk, as well as slides, online on the <a href="https://www.usenix.org/conference/nsdi12/resilient-distributed-datasets-fault-tolerant-abstraction-memory-cluster-computing">NSDI website</a>. diff --git a/news/_posts/2012-10-15-spark-version-0-6-0-released.html b/news/_posts/2012-10-15-spark-version-0-6-0-released.html new file mode 100644 index 000000000..365145490 --- /dev/null +++ b/news/_posts/2012-10-15-spark-version-0-6-0-released.html @@ -0,0 +1,13 @@ +--- +layout: post +title: Spark version 0.6.0 released +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '1' +--- +<a href="/spark-release-0-6-0">Spark version 0.6.0</a> was released today, a major release that brings a wide range of performance improvements and new features, including a simpler standalone deploy mode and a Java API. Read more about it in the <a href="/spark-release-0-6-0">release notes</a>. diff --git a/news/_posts/2012-11-22-spark-0-6-1-and-0-5-2-released.html b/news/_posts/2012-11-22-spark-0-6-1-and-0-5-2-released.html new file mode 100644 index 000000000..e1da4c3a0 --- /dev/null +++ b/news/_posts/2012-11-22-spark-0-6-1-and-0-5-2-released.html @@ -0,0 +1,13 @@ +--- +layout: post +title: Spark 0.6.1 and 0.5.2 out +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '1' +--- +Today we've made available two maintenance releases for Spark: <a href="http://spark-project.org/spark-release-0-6-1/" title="Spark Release 0.6.1">0.6.1</a> and <a href="http://spark-project.org/spark-release-0-5-2/" title="Spark Release 0.5.2">0.5.2</a>. They both contain important bug fixes as well as some new features, such as the ability to build against Hadoop 2 distributions. We recommend that users update to the latest version for their branch; for new users, we recommend <a href="http://spark-project.org/spark-release-0-6-1/" title="Spark Release 0.6.1">0.6.1</a>. diff --git a/news/_posts/2012-12-21-spark-and-shark-in-the-news.html b/news/_posts/2012-12-21-spark-and-shark-in-the-news.html new file mode 100644 index 000000000..d8f345787 --- /dev/null +++ b/news/_posts/2012-12-21-spark-and-shark-in-the-news.html @@ -0,0 +1,25 @@ +--- +layout: post +title: Spark and Shark in the news +categories: +- In-The-News +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +Recently, we've seen quite a bit of coverage of both Spark and <a href="http://shark.cs.berkeley.edu">Shark</a> in the news. I wanted to list some of the more recent articles, for readers interested in learning more.
+
+<ul>
+<li>Curt Monash, editor of the popular DBMS2 blog, wrote a great <a href="http://www.dbms2.com/2012/12/13/introduction-to-spark-shark-bdas-and-amplab/">introduction to Spark and Shark</a>, as well as a more detailed <a href="http://www.dbms2.com/2012/12/13/spark-shark-and-rdds-technology-notes/">technical overview</a>.</li>
+<li><a href="http://siliconangle.com/blog/2012/11/29/big-data-up-to-100x-faster-researchers-crank-up-the-speed-dial/">Silicon Angle</a> covered Spark and Shark after our <a href="https://amplab.cs.berkeley.edu/news/sparkshark-a-big-hit-at-aws-reinvent/">presentation at Amazon re:Invent</a>.</li>
+<li><a href="http://www.datanami.com/datanami/2012-12-01/the_week_in_big_data_research.html?page=2">Datanami</a> highlighted Shark in its survey of big data research projects.</li>
+<li>O'Reilly's Strata blog recently covered <a href="http://strata.oreilly.com/2012/08/seven-reasons-why-i-like-spark.html">Spark</a>, <a href="http://strata.oreilly.com/2012/11/shark-real-time-queries-and-analytics-for-big-data.html">Shark</a>, and the <a href="http://strata.oreilly.com/2012/10/spark-0-6-improves-performance-and-accessibility.html">Spark 0.6 release</a>.</li>
+<li><a href="http://data-informed.com/spark-an-open-source-engine-for-iterative-data-mining/">DataInformed</a> interviewed two Spark users and wrote about their applications in anomaly detection, predictive analytics and data mining.</li>
+</ul>
+
+In other news, there will be a full day of tutorials on Spark and Shark at the <a href="http://strataconf.com/strata2013">O'Reilly Strata conference</a> in February. They include a three-hour <a href="http://strataconf.com/strata2013/public/schedule/detail/27438">introduction to Spark, Shark and BDAS</a> Tuesday morning, and a three-hour <a href="http://strataconf.com/strata2013/public/schedule/detail/27440">hands-on exercise session</a>. diff --git a/news/_posts/2012-12-21-video-from-first-spark-development-meetup.html b/news/_posts/2012-12-21-video-from-first-spark-development-meetup.html new file mode 100644 index 000000000..ba45bc97c --- /dev/null +++ b/news/_posts/2012-12-21-video-from-first-spark-development-meetup.html @@ -0,0 +1,14 @@ +--- +layout: post +title: Video up from first Spark development meetup +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +On December 18th, we held the first of a series of Spark development meetups, for people interested in learning the Spark codebase and contributing to the project. There was quite a bit more demand than we anticipated, with over 80 people signing up and 64 attending. The first meetup was an <a href="http://www.meetup.com/spark-users/events/94101942/">introduction to Spark internals</a>. Thanks to one of the attendees, there's now a <a href="http://www.youtube.com/watch?v=49Hr5xZyTEA">video of the meetup</a> on YouTube. We've also posted the <a href="http://files.meetup.com/3138542/dev-meetup-dec-2012.pptx">slides</a>. Look to see more development meetups on Spark and Shark in the future. diff --git a/news/_posts/2013-01-12-spark-tips-from-quantifind.html b/news/_posts/2013-01-12-spark-tips-from-quantifind.html new file mode 100644 index 000000000..8afac5dd8 --- /dev/null +++ b/news/_posts/2013-01-12-spark-tips-from-quantifind.html @@ -0,0 +1,19 @@ +--- +layout: post +title: Spark tips from Quantifind +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +<a href="http://www.quantfind.com">Quantfind</a>, one of the Bay Area companies that has been using Spark for predictive analytics, recently posted two useful entries on working with Spark in their tech blog:
+<ul>
+<li><a href="http://blog.quantifind.com/posts/spark-unit-test/">Unit testing with Spark</a></li>
+<li><a href="http://blog.quantifind.com/posts/logging-post/">Configuring Spark's logs</a></li>
+</ul>
+Thanks for sharing this, and looking forward to see others! diff --git a/news/_posts/2013-02-07-spark-0-6-2-released.html b/news/_posts/2013-02-07-spark-0-6-2-released.html new file mode 100644 index 000000000..ae9146753 --- /dev/null +++ b/news/_posts/2013-02-07-spark-0-6-2-released.html @@ -0,0 +1,14 @@ +--- +layout: post +title: Spark 0.6.2 released +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +We recently released <a href="http://spark-project.org/spark-release-0-6-2/" title="Spark Release 0.6.2">Spark 0.6.2</a>, a new version of Spark. This is a maintenance release that includes several bug fixes and usability improvements (see the <a href="http://spark-project.org/spark-release-0-6-2/" title="Spark Release 0.6.2">release notes</a>). We recommend that all users upgrade to this release. diff --git a/news/_posts/2013-02-24-run-spark-and-shark-on-amazon-emr.html b/news/_posts/2013-02-24-run-spark-and-shark-on-amazon-emr.html new file mode 100644 index 000000000..cf86ec957 --- /dev/null +++ b/news/_posts/2013-02-24-run-spark-and-shark-on-amazon-emr.html @@ -0,0 +1,14 @@ +--- +layout: post +title: Spark/Shark Tutorial for Amazon EMR +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +This weekend, Amazon posted an <a href="http://aws.amazon.com/articles/Elastic-MapReduce/4926593393724923">article</a> and code that make it easy to launch Spark and Shark on Elastic MapReduce. The article includes examples of how to run both interactive Scala commands and SQL queries from Shark on data in S3. Head over to the <a href="http://aws.amazon.com/articles/Elastic-MapReduce/4926593393724923">Amazon article</a> for details. We're very excited because, to our knowledge, this makes Spark the first non-Hadoop engine that you can launch with EMR. diff --git a/news/_posts/2013-02-27-spark-0-7-0-released.html b/news/_posts/2013-02-27-spark-0-7-0-released.html new file mode 100644 index 000000000..de2e67e20 --- /dev/null +++ b/news/_posts/2013-02-27-spark-0-7-0-released.html @@ -0,0 +1,14 @@ +--- +layout: post +title: Spark 0.7.0 released +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +We're proud to announce the release of <a href="http://spark-project.org/spark-release-0-7-0/" title="Spark Release 0.7.0">Spark 0.7.0</a>, a new major version of Spark that adds several key features, including a <a href="/docs/latest/python-programming-guide.html">Python API</a> for Spark and an <a href="/docs/latest/streaming-programming-guide.html">alpha of Spark Streaming</a>. This release is the result of the largest group of contributors yet behind a Spark release -- 31 contributors from inside and outside Berkeley. Head over to the <a href="http://spark-project.org/spark-release-0-7-0/" title="Spark Release 0.7.0">release notes</a> to read more about the new features, or <a href="/downloads/">download</a> the release today. diff --git a/news/_posts/2013-03-17-strata-exercises-now-available-online.html b/news/_posts/2013-03-17-strata-exercises-now-available-online.html new file mode 100644 index 000000000..7ff0eda9d --- /dev/null +++ b/news/_posts/2013-03-17-strata-exercises-now-available-online.html @@ -0,0 +1,14 @@ +--- +layout: post +title: Strata exercises now available online +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +At this year's <a href="http://strataconf.com/strata2013">Strata</a> conference, the AMP Lab hosted a full day of tutorials on Spark, Shark, and Spark Streaming, including online exercises on Amazon EC2. Those exercises are now <a href="http://ampcamp.berkeley.edu/big-data-mini-course/">available online</a>, letting you learn Spark and Shark at your own pace on an EC2 cluster with real data. They are a great resource for learning the systems. You can also find <a href="http://ampcamp.berkeley.edu/amp-camp-two-strata-2013/">slides</a> from the Strata tutorials online, as well as <a href="http://ampcamp.berkeley.edu/amp-camp-one-berkeley-2012/">videos</a> from the AMP Camp workshop we held at Berkeley in August. diff --git a/news/_posts/2013-04-16-spark-screencasts-published.html b/news/_posts/2013-04-16-spark-screencasts-published.html new file mode 100644 index 000000000..96168d292 --- /dev/null +++ b/news/_posts/2013-04-16-spark-screencasts-published.html @@ -0,0 +1,20 @@ +--- +layout: post +title: Spark screencasts published +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '2' + _wpas_done_all: '1' +--- +We have released the first two screencasts in a series of short hands-on video training courses we will be publishing to help new users get up and running with Spark in minutes.
+
+The first Spark screencast is called <a href="http://spark-project.org/screencast-1-first-steps-with-spark/">First Steps With Spark</a> and walks you through downloading and building Spark, as well as using the Spark shell, all in less than 10 minutes!
+
+The second screencast is a 2 minute <a href="http://spark-project.org/spark-documentation-overview-screencast-2/">overview of the Spark documentation</a>.
+
+We hope you find these screencasts useful. diff --git a/news/_posts/2013-06-02-spark-0-7-2-released.html b/news/_posts/2013-06-02-spark-0-7-2-released.html new file mode 100644 index 000000000..82ae05345 --- /dev/null +++ b/news/_posts/2013-06-02-spark-0-7-2-released.html @@ -0,0 +1,14 @@ +--- +layout: post +title: Spark 0.7.2 released +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +We're happy to announce the release of <a href="http://spark-project.org/spark-release-0-7-2/" title="Spark Release 0.7.2">Spark 0.7.2</a>, a new maintenance release that includes several bug fixes and improvements, as well as new code examples and API features. We recommend that all users update to this release. Head over to the <a href="http://spark-project.org/spark-release-0-7-2/" title="Spark Release 0.7.2">release notes</a> to read about the new features, or <a href="/downloads/">download</a> the release today. diff --git a/news/_posts/2013-06-21-spark-accepted-into-apache-incubator.html b/news/_posts/2013-06-21-spark-accepted-into-apache-incubator.html new file mode 100644 index 000000000..1a6ca6dd8 --- /dev/null +++ b/news/_posts/2013-06-21-spark-accepted-into-apache-incubator.html @@ -0,0 +1,14 @@ +--- +layout: post +title: Spark accepted into Apache Incubator +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +Spark was recently <a href="http://mail-archives.apache.org/mod_mbox/incubator-general/201306.mbox/%3CCDE7B773.E9A48%25chris.a.mattmann%40jpl.nasa.gov%3E">accepted</a> into the <a href="http://incubator.apache.org">Apache Incubator</a>, which will serve as the long-term home for the project. While moving the source code and issue tracking to Apache will take some time, we are excited to be joining the community at Apache. Stay tuned on this site for updates on how the project hosting will change. diff --git a/news/_posts/2013-06-21-spark-featured-in-wired.html b/news/_posts/2013-06-21-spark-featured-in-wired.html new file mode 100644 index 000000000..0b3804390 --- /dev/null +++ b/news/_posts/2013-06-21-spark-featured-in-wired.html @@ -0,0 +1,15 @@ +--- +layout: post +title: Spark featured in Wired +categories: +- In-The-News +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +Spark, its creators at the AMP Lab, and some of its users were featured in a <a href="http://www.wired.com/wiredenterprise/2013/06/yahoo-amazon-amplab-spark/all/">Wired Enterprise article</a> a few days ago. Read on to learn a little about how Spark is being used in industry. diff --git a/news/_posts/2013-07-16-spark-0-7-3-released.html b/news/_posts/2013-07-16-spark-0-7-3-released.html new file mode 100644 index 000000000..862aab74e --- /dev/null +++ b/news/_posts/2013-07-16-spark-0-7-3-released.html @@ -0,0 +1,14 @@ +--- +layout: post +title: Spark 0.7.3 released +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +We've just posted <a href="http://spark-project.org/spark-release-0-7-3/" title="Spark Release 0.7.3">Spark Release 0.7.3</a>, a maintenance release that contains several fixes, including streaming API updates and new functionality for adding JARs to a <code>spark-shell</code> session. We recommend that all users update to this release. Visit the <a href="http://spark-project.org/spark-release-0-7-3/" title="Spark Release 0.7.3">release notes</a> to read about the new features, or <a href="/downloads/">download</a> the release today. diff --git a/news/_posts/2013-07-22-spark-mailing-lists-moving-to-apache.html b/news/_posts/2013-07-22-spark-mailing-lists-moving-to-apache.html new file mode 100644 index 000000000..d0cdb52d3 --- /dev/null +++ b/news/_posts/2013-07-22-spark-mailing-lists-moving-to-apache.html @@ -0,0 +1,25 @@ +--- +layout: post +title: Spark mailing lists moving to Apache +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +<p>As part of the Spark project's recent move to Apache, we are planning to migrate the mailing lists to Apache infrastructure this month, so that <strong>the existing Google groups will become read-only on September 1, 2013</strong>. To keep receiving updates about Spark or to participate in development discussions, please subscribe to the following lists:</p>
+
+<ul>
+ <li>
+ <a href="http://mail-archives.apache.org/mod_mbox/incubator-spark-user/">user@spark.incubator.apache.org</a> -- for usage questions, help, and announcements. <a href="mailto:user-subscribe@spark.incubator.apache.org?subject=(send%20this%20email%20to%20subscribe)">(subscribe)</a> <a href="http://mail-archives.apache.org/mod_mbox/incubator-spark-user/">(archives)</a>
+ </li>
+ <li>
+ <a href="http://mail-archives.apache.org/mod_mbox/incubator-spark-dev/">dev@spark.incubator.apache.org</a> -- for people who want to contribute code to Spark. <a href="mailto:dev-subscribe@spark.incubator.apache.org?subject=(send%20this%20email%20to%20subscribe)">(subscribe)</a> <a href="http://mail-archives.apache.org/mod_mbox/incubator-spark-dev/">(archives)</a>
+ </li>
+</ul>
+
+<p>Most users will probably want the User list, but individuals interested in contributing code to the project should also subscribe to the Dev list.</p> diff --git a/news/_posts/2013-07-23-amp-camp-2013-registration-ope.html b/news/_posts/2013-07-23-amp-camp-2013-registration-ope.html new file mode 100644 index 000000000..7be7cc0fe --- /dev/null +++ b/news/_posts/2013-07-23-amp-camp-2013-registration-ope.html @@ -0,0 +1,14 @@ +--- +layout: post +title: Registration open for AMP Camp training camp in Berkeley +categories: +- News +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +Want to learn how to use Spark, Shark, GraphX, and related technologies in person? The AMP Lab is hosting a two-day training workshop for them on August 29th and 30th in Berkeley. The workshop will include tutorials, talks from users, and over four hours of hands-on exercises. <a href="http://ampcamp.berkeley.edu/amp-camp-three-berkeley-2013/">Registration is now open on the AMP Camp website</a>, for a price of $250 per person. We recommend signing up early because last year's workshop was sold out. diff --git a/news/index.md b/news/index.md new file mode 100644 index 000000000..4434ecde7 --- /dev/null +++ b/news/index.md @@ -0,0 +1,15 @@ +--- +layout: global +type: archive +title: Spark News Archive +--- +## Spark News Archive +{% for post in site.categories.news %} + <article class="hentry"> + <header class="entry-header"> + <h1 class="entry-title"><a href="{{ post.url }}">{{ post.title }}</a></h1> + <div class="entry-meta">{{post.date | date: "%B %d, %Y"}}</div> + </header> + <div class="entry-content">{{post.excerpt}}</div> + </article> +{% endfor %} diff --git a/releases/_posts/2011-07-14-spark-release-0-3.md b/releases/_posts/2011-07-14-spark-release-0-3.md new file mode 100644 index 000000000..4238398f4 --- /dev/null +++ b/releases/_posts/2011-07-14-spark-release-0-3.md @@ -0,0 +1,62 @@ +---
+layout: post
+title: Spark Release 0.3
+categories:
+- Releases
+tags: []
+status: publish
+type: post
+published: true
+---
+Spark 0.3 brings a variety of new features. You can download it for either <a href="https://github.com/mesos/spark/tarball/0.3-scala-2.9">Scala 2.9</a> or <a href="https://github.com/mesos/spark/tarball/0.3-scala-2.8">Scala 2.8</a>.
+
+<h3>Scala 2.9 Support</h3>
+
+This is the first release to support Scala 2.9 in addition to 2.8. Future releases are likely to be 2.9-only unless there is high demand for 2.8.
+
+<h3>Save Operations</h3>
+
+You can now save distributed datasets to the Hadoop filesystem (HDFS), Amazon S3, Hypertable, and any other storage system supported by Hadoop. There are convenience methods for several common formats, like text files and SequenceFiles. For example, to save a dataset as text:
+
+<div class="code">
+<span class="keyword">val</span> numbers = spark.parallelize(1 to 100)<br> numbers.<span class="sparkop">saveAsTextFile</span>(<span class="string">"hdfs://..."</span>)
+</div>
+
+<h3>Native Types for SequenceFiles</h3>
+
+In working with SequenceFiles, which store objects that implement Hadoop's Writable interface, Spark will now let you use native types for certain common Writable types, like IntWritable and Text. For example:
+
+<div class="code">
+<span class="comment">// Will read a SequenceFile of (IntWritable, Text)</span><br>
+<span class="keyword">val</span> data = spark.sequenceFile[Int, String](<span class="string">"hdfs://..."</span>)
+</div>
+
+Similarly, you can save datasets of basic types directly as SequenceFiles:
+
+<div class="code">
+<span class="comment">// Will write a SequenceFile of (IntWritable, IntWritable)</span><br>
+<span class="keyword">val</span> squares = spark.parallelize(1 to 100).<span class="sparkop">map</span>(<span class="closure">n => (n, n*n)</span>)<br>
+squares.saveAsSequenceFile(<span class="string">"hdfs://..."</span>)
+</div>
+
+<h3>Maven Integration</h3>
+
+Spark now fetches dependencies via Maven and can publish Maven artifacts for easier dependency management.
+
+<h3>Faster Broadcast & Shuffle</h3>
+
+This release includes broadcast and shuffle algorithms from <a href="http://www.mosharaf.com/wp-content/uploads/orchestra-sigcomm11.pdf">this paper</a> to better support applications that communicate large amounts of data.
+
+<h3>Support for Non-Filesystem Hadoop Input Formats</h3>
+
+The new <tt>SparkContext.hadoopRDD</tt> method allows reading data from Hadoop-compatible storage systems other than file systems, such as HBase, Hypertable, etc.
+
+<h3>Other Features</h3>
+
+<ul>
+ <li>Outer join operators (<tt>leftOuterJoin</tt>, <tt>rightOuterJoin</tt>, etc).</li>
+ <li>Support for Scala 2.9 interpreter features (history search, Ctrl-C current line, etc) in the 2.9 version.</li>
+ <li>Better default levels of parallelism for various operations.</li>
+ <li>Ability to control number of splits in a file.</li>
+ <li>Various bug fixes.</li>
+</ul>
diff --git a/releases/_posts/2012-06-12-spark-release-0-5-0.md b/releases/_posts/2012-06-12-spark-release-0-5-0.md new file mode 100644 index 000000000..df27a0996 --- /dev/null +++ b/releases/_posts/2012-06-12-spark-release-0-5-0.md @@ -0,0 +1,36 @@ +--- +layout: post +title: Spark Release 0.5.0 +categories: +- Releases +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '1' +--- +Spark 0.5.0 brings several new features and sets the stage for some big changes coming this summer as we incorporate code from the <a href="http://www.cs.berkeley.edu/~matei/papers/2012/hotcloud_spark_streaming.pdf">Spark Streaming</a> project. You can download it as a <a href="https://github.com/mesos/spark/zipball/v0.5.0">zip</a> or <a href="https://github.com/mesos/spark/tarball/v0.5.0">tar.gz</a>.
+
+<h3>Mesos 0.9 Support</h3>
+
+This release runs on <a href="http://www.mesosproject.org/">Apache Mesos 0.9</a>, the first Apache Incubator release of Mesos, which contains significant usability and stability improvements. Most notable are better memory accounting for applications with long-term memory use, easier access of old jobs' traces and logs (by keeping a history of executed tasks on the web UI), and simpler installation.
+
+<h3>Performance Improvements</h3>
+Spark's scheduling is more communication-efficient when sending out operations on RDDs with large lineage graphs. In addition, the cache replacement policy has been improved to more smartly replace data when an RDD does not fit in the cache, shuffles are more efficient, and the serializer used for shipping closures is now configurable, making it possible to use faster libraries than Java serialization there.
+
+<h3>Debug Improvements</h3>
+
+Spark now reports exceptions on the worker nodes back to the master, so you can see them all in one log file. It also automatically marks and filters duplicate errors.
+
+<h3>New Operators</h3>
+
+These include <tt>sortByKey</tt> for parallel sorting, <tt>takeSample</tt>, and more efficient <tt>fold</tt> and <tt>aggregate</tt> operators. In addition, more of the old operators make use of, and retain, RDD partitioning information to reduce communication cost. For example, if you <tt>join</tt> two hash-partitioned RDDs that were partitioned in the same way, Spark will not shuffle any data across the network.
+
+<h3>EC2 Launch Script Improvements</h3>
+
+Spark's EC2 launch scripts are now included in the main package, and have the ability to discover and use the latest Spark AMI automatically instead of launching a hardcoded machine image ID.
+
+<h3>New Hadoop API Support</h3>
+
+You can now use Spark to read and write data to storage formats in the new <tt>org.apache.mapreduce</tt> packages (the "new Hadoop" API). In addition, this release fixes an issue caused by a HDFS initialization bug in some recent versions of HDFS. diff --git a/releases/_posts/2012-10-11-spark-release-0-5-1.md b/releases/_posts/2012-10-11-spark-release-0-5-1.md new file mode 100644 index 000000000..c5c935ed6 --- /dev/null +++ b/releases/_posts/2012-10-11-spark-release-0-5-1.md @@ -0,0 +1,46 @@ +--- +layout: post +title: Spark Release 0.5.1 +categories: +- Releases +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '1' +--- +Spark 0.5.1 is a maintenance release that adds several important bug fixes and usability features. You can download it as a <a href="http://github.com/downloads/mesos/spark/spark-0.5.1.tgz">tar.gz file</a>.
+
+<h3>Maven Publishing</h3>
+
+Spark is now available in Maven Central, making it easier to link into your programs without having to build it as a JAR. Use the following Maven identifiers to add it to a project:
+<ul>
+ <li>groupId: org.spark-project</li>
+ <li>artifactId: spark-core_2.9.2</li>
+ <li>version: 0.5.1</li>
+</ul>
+
+<h3>Scala 2.9.2</h3>
+
+Spark now builds against Scala 2.9.2 by default.
+
+<h3>Improved Accumulators</h3>
+
+The new Accumulable class generalizes Accumulators for the case when the type being accumulated is not the same as the types of elements being added (e.g. you wish to accumulate a collection, such as a Set, by adding individual elements). This interface is also more efficient in avoiding the creation of temporary objects. (Contributed by Imran Rashid.)
+
+<h3>Bug Fixes</h3>
+
+<ul>
+ <li>Spark's algorithm for estimating the sizes of objects (in order to manage memory correctly) has been improved
+ to handle JVMs with 32- vs 64-bit pointers and to measure objects more accurately. (Contributed by Shivaram Venkataraman.)</li>
+ <li>Improved algorithms for taking random samples out of datasets to avoid biases that could occur in the previous ones. (Suggested by Henry Milner.)</li>
+ <li>Improved load balancing across nodes in sort operations.</li>
+ <li>Fixed a shuffle bug that could cause reduce tasks to fail to receive a map task's full output.</li>
+ <li>Fixed a bug with locating custom KryoSerializers.</li>
+ <li>Reduced memory consumption of <tt>saveAsObjectFile</tt> when objects are large.</li>
+</ul>
+
+<h3>EC2 Improvements</h3>
+
+Spark's EC2 launch script now configures Spark's memory limit automatically based on the machine's available RAM. diff --git a/releases/_posts/2012-10-15-spark-release-0-6-0.md b/releases/_posts/2012-10-15-spark-release-0-6-0.md new file mode 100644 index 000000000..fb17f3037 --- /dev/null +++ b/releases/_posts/2012-10-15-spark-release-0-6-0.md @@ -0,0 +1,90 @@ +--- +layout: post +title: Spark Release 0.6.0 +categories: +- Releases +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' +--- +Spark 0.6.0 is a major release that brings several new features, architectural changes, and performance enhancements. The most visible additions are a standalone deploy mode, a Java API, and expanded documentation; but there are also numerous other changes under the hood, which improve performance in some cases by as much as 2x.
+
+You can download this release as either a <a href="http://github.com/downloads/mesos/spark/spark-0.6.0-sources.tar.gz">source package</a> (2 MB tar.gz) or <a href="http://github.com/downloads/mesos/spark/spark-0.6.0-prebuilt.tar.gz">prebuilt package</a> (48 MB tar.gz)
+
+<h3>Simpler Deployment</h3>
+
+In addition to running on Mesos, Spark now has a <a href="/docs/0.6.0/spark-standalone.html">standalone deploy mode</a> that lets you quickly launch a cluster without installing an external cluster manager. The standalone mode only needs Java installed on each machine, and Spark deployed to it.
+
+In addition, there is <a href="/docs/0.6.0/running-on-yarn.html">experimental support for running on YARN</a> (Hadoop NextGen), currently in a separate branch.
+
+<h3>Java API</h3>
+
+Java programmers can now use Spark through a new <a href="/docs/0.6.0/java-programming-guide.html">Java API layer</a>. This layer makes available all of Spark's features, including parallel transformations, distributed datasets, broadcast variables, and accumulators, in a Java-friendly manner.
+
+<h3>Expanded Documentation</h3>
+
+Spark's <a href="/docs/0.6.0/">documentation</a> has been expanded with a new <a href="/docs/0.6.0/quick-start.html">quick start guide</a>, additional deployment instructions, configuration guide, tuning guide, and improved <a href="/docs/0.6.0/api/core">Scaladoc</a> API documentation.
+
+<h3>Engine Changes</h3>
+
+Under the hood, Spark 0.6 has new, custom storage and communication layers brought in from the upcoming <a href="http://www.cs.berkeley.edu/~matei/papers/2012/hotcloud_spark_streaming.pdf">Spark Streaming</a> project. These can improve performance over past versions by as much as 2x. Specifically:
+
+<ul>
+ <li>A new communication manager using asynchronous Java NIO lets shuffle operations run faster, especially when sending large amounts of data or when jobs have many tasks.</li>
+ <li>A new storage manager supports per-dataset storage level settings (e.g. whether to keep the dataset in memory, deserialized, on disk, etc, or even replicated across nodes).</li>
+ <li>Spark's scheduler and control plane have been optimized to better support ultra-low-latency jobs (under 500ms) and high-throughput scheduling decisions.</li>
+</ul>
+
+<h3>New APIs</h3>
+
+<ul>
+ <li>This release adds the ability to control caching strategies on a per-RDD level, so that different RDDs may be stored in memory, on disk, as serialized bytes, etc. You can choose your storage level using the <a href="/docs/0.6.0/scala-programming-guide.html#rdd-persistence"><tt>persist()</tt> method</a> on RDD.</li>
+ <li>A new Accumulable class generalizes Accumulators for the case when the type being accumulated is not the same as the types of elements being added (e.g. you wish to accumulate a collection, such as a Set, by adding individual elements).</li>
+ <li>You can now dynamically add files or JARs that should be shipped to your workers with <tt>SparkContext.addFile/Jar</tt>.</li>
+ <li>More Spark operators (e.g. joins) support custom partitioners.</li>
+</ul>
+
+<h3>Enhanced Debugging</h3>
+
+Spark's log now prints which operation in your program each RDD and job described in your logs belongs to, making it easier to tie back to which parts of your code experience problems.
+
+<h3>Maven Artifacts</h3>
+
+Spark is now available in Maven Central, making it easier to link into your programs without having to build it as a JAR. Use the following Maven identifiers to add it to a project:
+
+<ul>
+ <li>groupId: org.spark-project</li>
+ <li>artifactId: spark-core_2.9.2</li>
+ <li>version: 0.6.0</li>
+</ul>
+
+<h3>Compatibility</h3>
+
+This release is source-compatible with Spark 0.5 programs, but you will need to recompile them against 0.6. In addition, the configuration for caching has changed: instead of having a <tt>spark.cache.class</tt> parameter that sets one caching strategy for all RDDs, you can now set a <a href="/docs/0.6.0/scala-programming-guide.html#rdd-persistence">per-RDD storage level</a>. Spark will warn if you try to set <tt>spark.cache.class</tt>.
+
+<h3>Credits</h3>
+
+Spark 0.6 was the work of a large set of new contributors from Berkeley and outside.
+
+<ul>
+ <li>Tathagata Das contributed the new communication layer, and parts of the storage layer.</li>
+ <li>Haoyuan Li contributed the new storage manager.</li>
+ <li>Denny Britz contributed the YARN deploy mode, key aspects of the standalone one, and several other features.</li>
+ <li>Andy Konwinski contributed the revamped documentation site, Maven publishing, and several API docs.</li>
+ <li>Josh Rosen contributed the Java API, and several bug fixes.</li>
+ <li>Patrick Wendell contributed the enhanced debugging feature and helped with testing and documentation.</li>
+ <li>Reynold Xin contributed numerous bug and performance fixes.</li>
+ <li>Imran Rashid contributed the new Accumulable class.</li>
+ <li>Harvey Feng contributed improvements to shuffle operations.</li>
+ <li>Shivaram Venkataraman improved Spark's memory estimation and wrote a memory tuning guide.</li>
+ <li>Ravi Pandya contributed Spark run scripts for Windows.
+ </li><li>Mosharaf Chowdhury provided several fixes to broadcast.</li>
+ <li>Henry Milner pointed out several bugs in sampling algorithms.</li>
+ <li>Ray Racine provided improvements to the EC2 scripts.</li>
+ <li>Paul Ruan and Bill Zhao helped with testing.</li>
+</ul>
+
+<p style="padding-top:5px;">Thanks also to all the Spark users who have diligently suggested features or reported bugs.</p> diff --git a/releases/_posts/2012-11-22-spark-release-0-5-2.md b/releases/_posts/2012-11-22-spark-release-0-5-2.md new file mode 100644 index 000000000..794f3521d --- /dev/null +++ b/releases/_posts/2012-11-22-spark-release-0-5-2.md @@ -0,0 +1,15 @@ +--- +layout: post +title: Spark Release 0.5.2 +categories: +- Releases +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '1' +--- +Spark 0.5.2 is a minor release, whose main addition is to allow Spark to compile against Hadoop 2 distributions. To do this, edit <code>project/SparkBuild.scala</code> and change both the <code>HADOOP_VERSION</code> and <code>HADOOP_MAJOR_VERSION</code> variables, then recompile Spark. This change was contributed by Thomas Dudziak.
+
+You can download Spark 0.5.2 as a <a href="https://github.com/downloads/mesos/spark/spark-0.5.2-sources.tgz">tar.gz file</a> (2 MB). diff --git a/releases/_posts/2012-11-22-spark-release-0-6-1.md b/releases/_posts/2012-11-22-spark-release-0-6-1.md new file mode 100644 index 000000000..6a0799429 --- /dev/null +++ b/releases/_posts/2012-11-22-spark-release-0-6-1.md @@ -0,0 +1,30 @@ +--- +layout: post +title: Spark Release 0.6.1 +categories: +- Releases +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' +--- +Spark 0.6.1 is a maintenance release that contains several important bug fixes and performance improvements. You can download it as a <a href="https://github.com/downloads/mesos/spark/spark-0.6.1-sources.tgz">source package</a> (2.4 MB tar.gz) or <a href="https://github.com/downloads/mesos/spark/spark-0.6.1-prebuilt.tgz">prebuilt package</a> (48 MB tar.gz).
+
+The fixes and improvements in this version include:
+<ul>
+ <li>Fixed overly aggressive message timeouts that could cause workers to disconnect from the cluster</li>
+ <li>Fixed a bug in the standalone deploy mode that did not expose hostnames to scheduler, affecting HDFS locality</li>
+ <li>Improved connection reuse in shuffle, which can greatly speed up small shuffles (contributed by Reynold Xin)</li>
+ <li>Fixed some potential deadlocks in the block manager (contributed by Tathagata Das)</li>
+ <li>Fixed a bug getting IDs of failed hosts from Mesos (contributed by Imran Rashid)</li>
+ <li>Several EC2 script improvements, like better handling of spot instances (contributed by Josh Rosen)</li>
+ <li>Made the local IP address that Spark binds to customizable (contributed by Mikhail Bautin)</li>
+ <li>Support for Hadoop 2 distributions (contributed by Thomas Dudziak)</li>
+ <li>Support for locating Scala on Debian distributions (contributed by Thomas Dudziak)</li>
+ <li>Improved standalone cluster web UI to show more information about jobs</li>
+ <li>Added an option to spread out jobs over the standalone cluster instead of concentrating them on a small number of nodes (<code>spark.deploy.spreadOut</code>)</li>
+</ul>
+
+We recommend that all Spark 0.6 users update to this maintenance release. diff --git a/releases/_posts/2013-02-07-spark-release-0-6-2.md b/releases/_posts/2013-02-07-spark-release-0-6-2.md new file mode 100644 index 000000000..e44c72a70 --- /dev/null +++ b/releases/_posts/2013-02-07-spark-release-0-6-2.md @@ -0,0 +1,43 @@ +--- +layout: post +title: Spark Release 0.6.2 +categories: [] +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +Spark 0.6.2 is a maintenance release that contains several bug fixes and usability improvements. You can download it as a <a href="http://spark-project.org/files/spark-0.6.2-sources.tgz">source package</a> (2.5 MB tar.gz) or <a href="http://spark-project.org/files/spark-0.6.2-prebuilt.tgz">prebuilt package</a> (48 MB tar.gz).
+
+We recommend that all Spark 0.6 users update to this maintenance release.
+
+The fixes and improvements in this version include:
+<ul>
+ <li>A number of fault tolerance fixes regarding detecting dead nodes, handling missing map output fetches, and allowing failed nodes to rejoin the cluster</li>
+ <li>Documentation fixes that clarify the configuration for the standalone mode and improve the quick start instructions</li>
+ <li>A connection reuse bug fix that improves shuffle performance</li>
+ <li>Support for launching a cluster across multiple availability zones in the EC2 scripts</li>
+ <li>Support for deleting security groups when an EC2 cluster is terminated</li>
+ <li>Improved memory configuration for the standalone deploy cluster daemons: instead of using <code>SPARK_MEM</code> for their memory, which often led people to give them much more memory than they intended, they now use a separate variable, <code>SPARK_DAEMON_MEMORY</code>, with a reasonable default of 512 MB
+ <li>Fixes to the Windows run scripts for Spark</li>
+ <li>Better detection of a machine's external IP address</li>
+ <li>Several small optimizations and bug fixes</li>
+</ul>
+
+In total, eleven people contributed to this release:
+<ul>
+ <li>Stephen Haberman (bug fix)</li>
+ <li>Shane Huang (shuffle fix)</li>
+ <li>Fernand Pajot (bug fix)</li>
+ <li>Andrew Psaltis (bug fix)</li>
+ <li>Imran Rashid (standalone cluster, bug fix)</li>
+ <li>Charles Reiss (fault recovery fixes, node re-registration, tests)</li>
+ <li>Josh Rosen (fault recovery, Java API fixes, deploy scripts)</li>
+ <li>Peter Sankauskas (EC2 scripts)</li>
+ <li>Lee Moon Soo (bug fix)</li>
+ <li>Patrick Wendell (bugs, docs)</li>
+ <li>Matei Zaharia (fault recovery, UI, docs, bug fixes)</li>
+</ul> diff --git a/releases/_posts/2013-02-27-spark-release-0-7-0.md b/releases/_posts/2013-02-27-spark-release-0-7-0.md new file mode 100644 index 000000000..ffb48a9e8 --- /dev/null +++ b/releases/_posts/2013-02-27-spark-release-0-7-0.md @@ -0,0 +1,112 @@ +--- +layout: post +title: Spark Release 0.7.0 +categories: [] +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +The Spark team is proud to release version 0.7.0, a new major release that brings several new features. Most notable are a <a href="/docs/0.7.0/python-programming-guide.html">Python API for Spark</a> and an <a href="/docs/0.7.0/streaming-programming-guide.html">alpha of Spark Streaming</a>. (Details on Spark Streaming can also be found in this <a href="http://www.eecs.berkeley.edu/Pubs/TechRpts/2012/EECS-2012-259.pdf">technical report</a>.) The release also adds numerous other improvements across the board. Overall, this is our biggest release to date, with 31 contributors, of which 20 were external to Berkeley.
+
+You can download Spark 0.7.0 as either a <a href="/files/spark-0.7.0-sources.tgz">source package</a> (4 MB tar.gz) or <a href="/files/spark-0.7.0-prebuilt.tgz">prebuilt package</a> (60 MB tar.gz).
+
+<h3>Python API</h3>
+
+Spark 0.7 adds a <a href="/docs/0.7.0/python-programming-guide.html">Python API</a> called PySpark that makes it possible to use Spark from Python, both in standalone programs and in interactive Python shells. It uses the standard CPython runtime, so your programs can call into native libraries like NumPy and SciPy. Like the Scala and Java APIs, PySpark will automatically ship functions from your main program, along with the variables they depend on, to the cluster. PySpark supports most Spark features, including RDDs, accumulators, broadcast variables, and HDFS input and output.
+
+<h3>Spark Streaming Alpha</h3>
+
+Spark Streaming is a new extension of Spark that adds near-real-time processing capability. It offers a simple and high-level API, where users can transform streams using parallel operations like <tt>map</tt>, <tt>filter</tt>, <tt>reduce</tt>, and new sliding window functions. It automatically distributes work over a cluster and provides efficient fault recovery with exactly-once semantics for transformations, without relying on costly transactions to an external system. Spark Streaming is described in more detail in <a href="/talks/strata_spark_streaming.ppt">these slides</a> and <a href="http://www.eecs.berkeley.edu/Pubs/TechRpts/2012/EECS-2012-259.pdf">our technical report</a>. This release is our first alpha of Spark Streaming, with most of the functionality implemented and APIs in Java and Scala.
+
+<h3>Memory Dashboard</h3>
+
+Spark jobs now launch a web dashboard for monitoring the memory usage of each distributed dataset (RDD) in the program. Look for lines like this in your log:
+
+<tt>15:08:44 INFO BlockManagerUI: Started BlockManager web UI at http://mbk.local:63814</tt>
+
+You can also control which port to use through the <tt>spark.ui.port</tt> property.
+
+<h3>Maven Build</h3>
+
+Spark can now be built using Maven in addition to SBT. The Maven build enables easier publishing to repositories of your choice, easy selection of Hadoop versions using the Maven profile (<tt>-Phadoop1</tt> or <tt>-Phadoop2</tt>), as well as Debian packaging using <tt>mvn -Phadoop1,deb install</tt>.
+
+<h3>New Operations</h3>
+
+This release adds several RDD transformations, including <tt>keys</tt>, <tt>values</tt>, <tt>keyBy</tt>, <tt>subtract</tt>, <tt>coalesce</tt>, <tt>zip</tt>. It also adds <tt>SparkContext.hadoopConfiguration</tt> to allow programs to configure Hadoop input/output settings globally across operations. Finally, it adds the <tt>RDD.toDebugString()</tt> method, which can be used to print an RDD's lineage graph for troubleshooting.
+
+<h3>EC2 Improvements</h3>
+
+<ul>
+ <li>Spark will now read S3 credentials from the <tt>AWS_ACCESS_KEY_ID</tt> and <tt>AWS_SECRET_ACCESS_KEY</tt> environment variables, if set, making it easier to access Amazon S3.</li>
+ <li>This release fixes a bug with S3 access that would leave streams open when they are not fully read (e.g. when calling <tt>RDD.first()</tt> or a SQL query with a limit), causing nodes to hang.</li>
+ <li>The EC2 scripts now support both standalone and Mesos clusters, and launch Ganglia on the cluster.</li>
+ <li>Spark EC2 clusters can now be spread across multiple availability zones.</li>
+</ul>
+
+<h3>Other Improvements</h3>
+
+<ul>
+ <li>Shuffle operations like <tt>groupByKey</tt> and <tt>reduceByKey</tt> now try to infer parallelism from the size of the parent RDD (unless <tt>spark.default.parallelism</tt> is set).</li>
+ <li>Several performance improvements to shuffles.</li>
+ <li>Standalone deploy cluster now spreads jobs out across machines by default, leading to better data locality.</li>
+ <li>Better error reporting when jobs aren't being launched due to not enough resources.</li>
+ <li>Standalone deploy web UI now includes JSON endpoints for querying cluster state.</li>
+ <li>Better support for IBM JVM.</li>
+ <li>Default Hadoop version dependency updated to 1.0.4.</li>
+ <li>Improved failure handling and reporting of error messages.</li>
+ <li>Separate configuration for standalone cluster daemons and user applications.</li>
+ <li>Significant refactoring of the scheduler codebase to enable richer unit testing.</li>
+ <li>Several bug and performance fixes throughout.</li>
+</ul>
+
+<h3>Compatibility</h3>
+
+This release is API-compatible with Spark 0.6 programs, but the following features changed slightly:
+<ul>
+ <li>Parallel shuffle operations where you don't specify a level of parallelism use the number of partitions of the parent RDD instead of a constant default. However, if you set <tt>spark.default.parallelism</tt>, they will use that.</li>
+ <li><tt>SparkContext.addFile</tt>, which distributes a file to worker nodes, is no longer guaranteed to put it in the executor's working directory---instead, you can find the directory it used using <tt>SparkFiles.getRootDirectory</tt>, or get a particular file using <tt>SparkFiles.get</tt>. This was done to avoid cluttering the local directory when running in local mode.</li>
+</ul>
+
+<h3>Credits</h3>
+
+Spark 0.7 was the work of many contributors from Berkeley and outside---in total, 31 different contributors, of which 20 were from outside Berkeley. Here are the people who contributed, along with areas they worked on:
+
+<ul>
+ <li>Mikhail Bautin -- Maven build</li>
+ <li>Denny Britz -- memory dashboard, streaming, bug fixes</li>
+ <li>Paul Cavallaro -- error reporting</li>
+ <li>Tathagata Das -- streaming (lead developer), 24/7 operation, bug fixes, docs</li>
+ <li>Thomas Dudziak -- Maven build, Hadoop 2 support</li>
+ <li>Harvey Feng -- bug fix</li>
+ <li>Stephen Haberman -- new RDD operations, configuration, S3 improvements, code cleanup, bug fixes</li>
+ <li>Tyson Hamilton -- JSON status endpoints</li>
+ <li>Mark Hamstra -- API improvements, docs</li>
+ <li>Michael Heuer -- docs</li>
+ <li>Shane Huang -- shuffle performance fixes</li>
+ <li>Andy Konwinski -- docs</li>
+ <li>Ryan LeCompte -- streaming</li>
+ <li>Haoyuan Li -- streaming</li>
+ <li>Richard McKinley -- build</li>
+ <li>Sean McNamara -- streaming</li>
+ <li>Lee Moon Soo -- bug fix</li>
+ <li>Fernand Pajot -- bug fix</li>
+ <li>Nick Pentreath -- Python API, examples</li>
+ <li>Andrew Psaltis -- bug fixes</li>
+ <li>Imran Rashid -- memory dashboard, bug fixes</li>
+ <li>Charles Reiss -- fault recovery fixes, code cleanup, testability, error reporting</li>
+ <li>Josh Rosen -- Python API (lead developer), EC2 scripts, bug fixes</li>
+ <li>Peter Sankauskas -- EC2 scripts</li>
+ <li>Prashant Sharma -- streaming</li>
+ <li>Shivaram Venkataraman -- EC2 scripts, optimizations</li>
+ <li>Patrick Wendell -- streaming, bug fixes, examples, docs</li>
+ <li>Reynold Xin -- optimizations, UI</li>
+ <li>Haitao Yao -- run scripts</li>
+ <li>Matei Zaharia -- streaming, fault recovery, Python API, code cleanup, bug fixes, docs</li>
+ <li>Eric Zhang -- examples</li>
+</ul>
+
+Thanks to everyone who contributed! diff --git a/releases/_posts/2013-06-02-spark-release-0-7-2.md b/releases/_posts/2013-06-02-spark-release-0-7-2.md new file mode 100644 index 000000000..9b1ed38f0 --- /dev/null +++ b/releases/_posts/2013-06-02-spark-release-0-7-2.md @@ -0,0 +1,56 @@ +---
+layout: post
+title: Spark Release 0.7.2
+categories: []
+tags: []
+status: publish
+type: post
+published: true
+meta:
+ _edit_last: '4'
+ _wpas_done_all: '1'
+---
+Spark 0.7.2 is a maintenance release that contains multiple bug fixes and improvements. You can download it as a <a href="http://spark-project.org/download-spark-0.7.2-sources">source package</a> (4 MB tar.gz) or get prebuilt packages for <a href="http://spark-project.org/download-spark-0.7.2-prebuilt-hadoop1">Hadoop 1 / CDH3</a> or <a href="http://spark-project.org/download-spark-0.7.2-prebuilt-cdh4">CDH 4</a> (61 MB tar.gz).
+
+
+We recommend that all users update to this maintenance release.
+
+
+The fixes and improvements in this version include:
+<ul>
+ <li>Scala version updated to 2.9.3.</li>
+ <li>Several improvements to Bagel, including performance fixes and a configurable storage level.</li>
+ <li>New API methods: subtractByKey, foldByKey, mapWith, filterWith, foreachPartition, and others.</li>
+ <li>A new metrics reporting interface, SparkListener, to collect information about each computation stage: task lengths, bytes shuffled, etc.</li>
+ <li>Several new examples using the Java API, including K-means and computing pi.</li>
+ <li>Support for launching multiple worker instances per host in the standalone mode.</li>
+ <li>Various bug fixes across the board.</li>
+</ul>
+
+The following people contributed to this release:
+<ul>
+ <li>Jey Kottalam (Maven build, bug fixes, EC2 scripts, packaging the release)</li>
+ <li>Andrew Ash (bug fixes, docs)</li>
+ <li>Andrey Kouznetsov (bug fixes)</li>
+ <li>Andy Konwinski (docs)</li>
+ <li>Charles Reiss (bug fixes)</li>
+ <li>Christoph Grothaus (bug fixes)</li>
+ <li>Erik van Oosten (bug fixes)</li>
+ <li>Giovanni Delussu (bug fixes)</li>
+ <li>Hiral Patel (bug fixes)</li>
+ <li>Holden Karau (error reporting, EC2 scripts)</li>
+ <li>Imran Rashid (metrics reporting system)</li>
+ <li>Josh Rosen (EC2 scripts)</li>
+ <li>Mark Hamstra (new API methods, tests)</li>
+ <li>Mikhail Bautin (build)</li>
+ <li>Mosharaf Chowdhury (bug fixes)</li>
+ <li>Nick Pentreath (Bagel, examples)</li>
+ <li>Patrick Wendell (bug fixes)</li>
+ <li>Reynold Xin (bug fixes)</li>
+ <li>Stephen Haberman (bug fixes, tests, subtractByKey)</li>
+ <li>Kalpit Shah (build, multiple workers per host)</li>
+ <li>Mike Potts (run scripts)</li>
+ <li>Matei Zaharia (Bagel, bug fixes, build)</li>
+</ul>
+
+We thank everyone who helped with this release, and hope to see more contributions from you in the future!
diff --git a/releases/_posts/2013-07-16-spark-release-0-7-3.md b/releases/_posts/2013-07-16-spark-release-0-7-3.md new file mode 100644 index 000000000..39b35fb8f --- /dev/null +++ b/releases/_posts/2013-07-16-spark-release-0-7-3.md @@ -0,0 +1,49 @@ +--- +layout: post +title: Spark Release 0.7.3 +categories: +- Releases +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '4' + _wpas_done_all: '1' +--- +Spark 0.7.3 is a maintenance release with several bug fixes, performance fixes, and new features. You can download it as a <a href="/download/spark-0.7.3-sources.tgz">source package</a> (4 MB tar.gz) or get prebuilt packages for <a href="/download/spark-0.7.3-prebuilt-hadoop1.tgz">Hadoop 1 / CDH3</a> or for <a href="/download/spark-0.7.3-prebuilt-cdh4.tgz">CDH 4</a> (61 MB tar.gz).
+
+We recommend that all users update to this maintenance release.
+
+The improvements in this release include:
+
+<ul>
+ <li><b>New "add JARs" functionality in Spark shell:</b> Users of <code>spark-shell</code> can now set the <code>ADD_JARS</code> environment variable to add a list of JARs to their clusters; these will also be sent to workers.</li>
+ <li><b>Windows fixes:</b> Spark standalone clusters now properly kill executors when a job ends or fails. In addition, adding JAR paths with backslashes will now work correctly.</li>
+ <li><b>Streaming API fixes:</b> The Kafka and Twitter APIs for Spark Streaming have been updated. In the Twitter case, this is to deal with the username/password authentication method being disabled in by Twitter, while in the Kafka case, it is to allow receiving messages other than strings. Note that these are breaking API changes as the Streaming API is still in alpha.</li>
+ <li><b>Python performance:</b> Spark's mechanism for spawning Python VMs has been improved to do so faster when the JVM has a large heap size, speeding up the Python API.</li>
+ <li><b>Mesos fixes:</b> JARs added to your job will now be on the classpath when deserializing task results in Mesos.</li>
+ <li><b>Error reporting:</b> Better error reporting for non-serializable exceptions and overly large task results.</li>
+ <li><b>Examples:</b> Added an example of stateful stream processing with <code>updateStateByKey</code>.</li>
+ <li><b>Build:</b> Spark Streaming no longer depends on the Twitter4J repo, which should allow it to build in China.</li>
+ <li><b>Bug fixes</b> in <code>foldByKey</code>, streaming <code>count</code>, statistics methods, documentation, and web UI.</li>
+</ul>
+
+The following people contributed to this release:
+
+<ul>
+ <li>Charles Reiss (Mesos)</li>
+ <li>Christoph Grothaus (Windows spawn fixes)</li>
+ <li>Christopher Nguyen (bug fixes)</li>
+ <li>James Phillpotts (Twitter input stream)</li>
+ <li>Jey Kottalam (Python performance)</li>
+ <li>Josh Rosen (usability)</li>
+ <li>Konstantin Boudnik (build)</li>
+ <li>Mark Hamstra (build)</li>
+ <li>Matei Zaharia (Windows, docs, ADD_JARS, Python, streaming)</li>
+ <li>Patrick Wendell (usability)</li>
+ <li>Tathagata Das (streaming fixes)</li>
+ <li>Jerry Shao (bug fixes)</li>
+ <li>S. Kumar (examples)</li>
+ <li>Sean McNamara (Kafka input streams, streaming fixes)</li>
+</ul> diff --git a/research.md b/research.md new file mode 100644 index 000000000..96c9a24ac --- /dev/null +++ b/research.md @@ -0,0 +1,53 @@ +--- +layout: global +title: Research +type: "page singular" +navigation: + weight: 6 + show: true +--- +<h2>Spark Research</h2> + +<p> +Spark started as a research project at UC Berkeley in the <a href="https://amplab.cs.berkeley.edu">AMPLab</a>, which focuses on big data analytics. +</p> + +<p class="noskip"> +Our goal was to design a programming model that supports a much wider class of applications than MapReduce, while maintaining its automatic fault tolerance. In particular, MapReduce is inefficient for <em>multi-pass</em> applications that require low-latency data sharing across multiple parallel operations. These applications are quite common in analytics, and include: +</p> + +<ul> + <li><em>Iterative algorithms</em>, including many machine learning algorithms and graph algorithms like PageRank.</li> + <li><em>Interactive data mining</em>, where a user would like to load data into RAM across a cluster and query it repeatedly.</li> + <li><em>OLAP reports</em> that run multiple aggregation queries on the same data.</li> +</ul> + +<p> +MapReduce and Dryad are suboptimal for these applications because they are based on acyclic data flow: an application has to run as a series of distinct jobs, each of which reads data from stable storage (e.g. a distributed file system) and writes it back to stable storage. They incur significant cost loading the data on each step and writing it back to replicated storage. +</p> + +<p> +Spark offers an abstraction called <a href="http://www.cs.berkeley.edu/~matei/papers/2012/nsdi_spark.pdf"><em>resilient distributed datasets (RDDs)</em></a> to support these applications efficiently. RDDs can be stored in memory between queries <em>without</em> requiring replication. Instead, they rebuild lost data on failure using <em>lineage</em>: each RDD remembers how it was built from other datasets (by transformations like <em>map</em>, <em>join</em> or <em>group-by</em>) to rebuild itself. RDDs allow Spark to outperform existing models by up to 100x in multi-pass analytics. We showed that RDDs can support a wide variety of iterative algorithms, as well as interactive data mining and a highly efficient SQL engine (the <a href="http://shark.cs.berkeley.edu">Shark</a> project). +</p> + +<p class="noskip">You can find more about the research behind Spark in our papers:</p> + +<ul> + <li> + <a href="http://www.eecs.berkeley.edu/Pubs/TechRpts/2012/EECS-2012-214.pdf">Shark: SQL and Rich Analytics at Scale</a>. Reynold Xin, Joshua Rosen, Matei Zaharia, Michael J. Franklin, Scott Shenker, Ion Stoica. <em>Technical Report UCB/EECS-2012-214</em>. November 2012. + </li> + <li> + <a href="http://www.cs.berkeley.edu/~matei/papers/2012/hotcloud_spark_streaming.pdf">Discretized Streams: An Efficient and Fault-Tolerant Model for Stream Processing on Large Clusters</a>. Matei Zaharia, Tathagata Das, Haoyuan Li, Scott Shenker, Ion Stoica. <em>HotCloud 2012</em>. June 2012. + </li> + <li> + <a href="http://www.cs.berkeley.edu/~matei/papers/2012/sigmod_shark_demo.pdf">Shark: Fast Data Analysis Using Coarse-grained Distributed Memory</a> (demo). Cliff Engle, Antonio Lupher, Reynold Xin, Matei Zaharia, Haoyuan Li, Scott Shenker, Ion Stoica. <em>SIGMOD 2012</em>. May 2012. <b>Best Demo Award</b>. + </li> + <li> + <a href="http://www.cs.berkeley.edu/~matei/papers/2012/nsdi_spark.pdf">Resilient Distributed Datasets: A Fault-Tolerant Abstraction for In-Memory Cluster Computing</a>. Matei Zaharia, Mosharaf Chowdhury, Tathagata Das, Ankur Dave, Justin Ma, Murphy McCauley, Michael J. Franklin, Scott Shenker, Ion Stoica. <em>NSDI 2012</em>. April 2012. <b>Best Paper Award</b> and <b>Honorable Mention for Community Award</b>. + </li> + <li> + <a href="http://www.cs.berkeley.edu/~matei/papers/2011/tr_spark.pdf">Resilient Distributed Datasets: A Fault-Tolerant Abstraction for In-Memory Cluster Computing</a>. Matei Zaharia, Mosharaf Chowdhury, Tathagata Das, Ankur Dave, Justin Ma, Murphy McCauley, Michael J. Franklin, Scott Shenker, Ion Stoica. <em>Technical Report UCB/EECS-2011-82</em>. July 2011.</li> + <li> + <a href="http://www.cs.berkeley.edu/~matei/papers/2010/hotcloud_spark.pdf">Spark: Cluster Computing with Working Sets</a>. Matei Zaharia, Mosharaf Chowdhury, Michael J. Franklin, Scott Shenker, Ion Stoica. <em>HotCloud 2010</em>. June 2010. + </li> +</ul> diff --git a/screencasts/_posts/2013-04-10-1-first-steps-with-spark.html b/screencasts/_posts/2013-04-10-1-first-steps-with-spark.html new file mode 100644 index 000000000..df4b26920 --- /dev/null +++ b/screencasts/_posts/2013-04-10-1-first-steps-with-spark.html @@ -0,0 +1,27 @@ +--- +layout: post +title: 'First Steps with Spark - Screencast #1' +categories: +- Screencasts +tags: [] +status: publish +type: post +published: true +meta: + _edit_last: '2' + _wpas_done_all: '1' + _wp_old_slug: first-steps-with-spark-screencast-1 +--- +This screencast marks the beginning of a series of hands-on screencasts we will be publishing to help new users get up and running in minutes. In this screencast, we: +<ol> + <li>Download and build Spark on a local machine (running OS X, but should be a similar process for Linux or Unix).</li> + <li>Introduce the API using the Spark interactive shell to explore a file.</li> +</ol> + +<div class="video-container shadow"><iframe width="755" height="705" src="http://www.youtube.com/embed/KYlLglXD6Ic?autohide=0&showinfo=0" frameborder="0" allowfullscreen></iframe></div> + +Check out the next spark screencast in the series, <a href="{{site.url}}screencasts/2-spark-documentation-overview.html">Spark Screencast #2 - Overview of Spark Documentation</a>. You can also find the Spark documentation online: +<ul> + <li><a href="{{site.url}}documentation.html">Spark documentation page</a></li> + <li><a href="http://ampcamp.berkeley.edu/big-data-mini-course-home">Amp Camp Mini Course</a></li> +</ul> diff --git a/screencasts/_posts/2013-04-11-2-spark-documentation-overview.html b/screencasts/_posts/2013-04-11-2-spark-documentation-overview.html new file mode 100644 index 000000000..7f5eb8160 --- /dev/null +++ b/screencasts/_posts/2013-04-11-2-spark-documentation-overview.html @@ -0,0 +1,19 @@ +--- +layout: post +title: 'Spark Documentation Overview – Screencast #2' +categories: +- Screencasts +tags: [] +status: publish +type: post +published: true +--- +This is our 2nd Spark screencast. In it, we take a tour of the documentation available for Spark users online. + +<div class="video-container shadow"><iframe width="755" height="705" src="http://www.youtube.com/embed/TikdEfsrFnw?autohide=0&showinfo=0" frameborder="0" allowfullscreen></iframe></div> + +And for convenience, here are links to the documentation shown in the video: +<ul> + <li><a href="{{site.url}}documentation.html">Spark documentation page</a></li> + <li><a href="http://ampcamp.berkeley.edu/big-data-mini-course-home">Amp Camp Mini Course</a></li> +</ul> diff --git a/screencasts/_posts/2013-04-16-3-transformations-and-caching.html b/screencasts/_posts/2013-04-16-3-transformations-and-caching.html new file mode 100644 index 000000000..dd05d0ce6 --- /dev/null +++ b/screencasts/_posts/2013-04-16-3-transformations-and-caching.html @@ -0,0 +1,14 @@ +--- +layout: post +title: 'Transformations and Caching - Spark Screencast #3' +categories: [] +tags: [] +status: publish +type: post +published: true +--- +In this third Spark screencast, we demonstrate more advanced use of RDD actions and transformations, as well as caching RDDs in memory. + +<div class="video-container shadow"><iframe width="755" height="705" src="http://www.youtube.com/embed/T1lZcimvL18?autohide=0&showinfo=0" frameborder="0" allowfullscreen></iframe></div> + +For more information, check out the <a href="{{site.url}}documentation.html">Spark documentation page</a>. diff --git a/site/index.html b/site/index.html index 470c467a5..cfd0f4da7 100644 --- a/site/index.html +++ b/site/index.html @@ -1 +1,181 @@ -<html><head><title>Apache Spark</title></head><body><h3>Future home of Apache Spark (Incubating).</h3><p>Test</p></body></html> +<!DOCTYPE html> +<!--[if IE 6]> +<html id="ie6" dir="ltr" lang="en-US"> +<![endif]--> +<!--[if IE 7]> +<html id="ie7" dir="ltr" lang="en-US"> +<![endif]--> +<!--[if IE 8]> +<html id="ie8" dir="ltr" lang="en-US"> +<![endif]--> +<!--[if !(IE 6) | !(IE 7) | !(IE 8) ]><!--> +<html dir="ltr" lang="en-US"> +<!--<![endif]--> +<head> + <link rel="shortcut icon" href="favicon.ico" /> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width" /> + <title>Home | Spark</title> + + <link rel="stylesheet" type="text/css" media="all" href="/css/style.css" /> + <link rel="stylesheet" href="/css/pygments-default.css"> + + <script type="text/javascript">//<![CDATA[ + // Google Analytics for WordPress by Yoast v4.2.8 | http://yoast.com/wordpress/google-analytics/ + var _gaq = _gaq || []; + _gaq.push(['_setAccount', 'UA-32518208-1']); + _gaq.push(['_trackPageview']); + (function () { + var ga = document.createElement('script'); + ga.type = 'text/javascript'; + ga.async = true; + ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; + var s = document.getElementsByTagName('script')[0]; + s.parentNode.insertBefore(ga, s); + })(); + //]]></script> + + <link rel='canonical' href='/index.html' /> + + <style type="text/css"> + #site-title, + #site-description { + position: absolute !important; + clip: rect(1px 1px 1px 1px); /* IE6, IE7 */ + clip: rect(1px, 1px, 1px, 1px); + } + </style> + <style type="text/css" id="custom-background-css"> + body.custom-background { background-color: #f1f1f1; } + </style> +</head> + +<!--body class="page two-column right-sidebar"--> +<body class="page"> +<div id="page" class="hfeed"> + + <header id="branding" role="banner"> + <hgroup> + <h1 id="site-title"><span><a href="/" title="Spark" rel="home">Spark</a></span></h1> + <h2 id="site-description">Lightning-Fast Cluster Computing</h2> + </hgroup> + + <a href="/"> + <img src="/images/spark-project-header1.png" width="1000" height="220" alt="Spark: Lightning-Fast Cluster Computing" title="Spark: Lightning-Fast Cluster Computing" /> + </a> + + <nav id="access" role="navigation"> + <h3 class="assistive-text">Main menu</h3> + <div class="menu-main-menu-container"> + <ul id="menu-main-menu" class="menu"> + + <li class="menu-item menu-item-type-post_type menu-item-object-page current-menu-item"> + <a href="/index.html">Home</a> + </li> + + <li class="menu-item menu-item-type-post_type menu-item-object-page "> + <a href="/downloads.html">Downloads</a> + </li> + + <li class="menu-item menu-item-type-post_type menu-item-object-page "> + <a href="/documentation.html">Documentation</a> + </li> + + <li class="menu-item menu-item-type-post_type menu-item-object-page "> + <a href="/examples.html">Examples</a> + </li> + + <li class="menu-item menu-item-type-post_type menu-item-object-page "> + <a href="/mailing-lists.html">Mailing Lists</a> + </li> + + <li class="menu-item menu-item-type-post_type menu-item-object-page "> + <a href="/research.html">Research</a> + </li> + + <li class="menu-item menu-item-type-post_type menu-item-object-page "> + <a href="/faq.html">FAQ</a> + </li> + + </ul></div> + </nav><!-- #access --> +</header><!-- #branding --> + + + + <div id="main"> + <div id="primary"> + <div id="content" role="main"> + + <article class="page type-page status-publish hentry"> + <h2 id="what-is-spark">What is Spark?</h2> +<p>Spark is an open source cluster computing system that aims to make data analytics <em>fast</em> — both fast to run and fast to write. +To run programs faster, Spark provides primitives for in-memory cluster computing: your job can load data into memory and query it repeatedly much more quickly than with disk-based systems like Hadoop MapReduce. +To make programming faster, Spark provides clean, concise APIs in <a href="http://www.scala-lang.org" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://www.scala-lang.org']);">Scala</a>, <a href="/docs/latest/quick-start.html#a-standalone-job-in-java">Java</a> and <a href="/docs/latest/quick-start.html#a-standalone-job-in-python">Python</a>. You can also use Spark interactively from the Scala and Python shells to rapidly query big datasets.</p> + +<h2 id="what-can-it-do">What can it do?</h2> +<p>Spark was initially developed for two applications where keeping data in memory helps: <em>iterative</em> algorithms, which are common in machine learning, and <em>interactive</em> data mining. In both cases, Spark can run up to <b>100x</b> faster than Hadoop MapReduce. However, you can use Spark for general data processing too. Check out our <a href="/examples.html">example jobs</a>. +Spark is also the engine behind <a href="http://shark.cs.berkeley.edu" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://shark.cs.berkeley.edu']);">Shark</a>, a fully <a href="http://hive.apache.org" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://hive.apache.org']);">Apache Hive</a>-compatible data warehousing system that can run 100x faster than Hive. +While Spark is a new engine, it can access any data source supported by Hadoop, making it easy to run over existing data.</p> + +<h2 id="who-uses-it">Who uses it?</h2> +<p>Spark was developed in the <a href="https://amplab.cs.berkeley.edu" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://amplab.cs.berkeley.edu']);">UC Berkeley AMPLab</a>. It’s used by several groups of researchers at Berkeley to run large-scale applications such as spam filtering and traffic prediction. It’s also used to accelerate data analytics at <a href="http://www.yahoo.com" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://www.yahoo.com']);">Yahoo!</a>, <a href="http://www.conviva.com" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://www.conviva.com']);">Conviva</a>, <a href="http://www.quantifind.com" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://www.quantifind.com']);">Quantifind</a>, and other companies — in total, 17 companies have contributed to Spark! Spark is <a href="https://github.com/mesos/spark" onclick="javascript:_gaq.push(['_trackEvent','outbound-article','http://github.com']);">open source</a> under a BSD license, so <a href="/downloads.html">download</a> it to check it out.</p> + +<h2 id="apache-incubator-notice">Apache Incubator notice</h2> +<p>Apache Spark is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.</p> + + + </article><!-- #post --> + + </div><!-- #content --> + + <div id="secondary" class="widget-area" role="complementary"> + +<h3 class="widget-title">Latest News</h3> +<div class="latestnewswidget"> + + <div><a href="/news/amp-camp-2013-registration-ope.html">Registration open for AMP Camp training camp in Berkeley</a> <span class="post-info">(July 23, 2013)</span></div> + + <div><a href="/news/spark-mailing-lists-moving-to-apache.html">Spark mailing lists moving to Apache</a> <span class="post-info">(July 22, 2013)</span></div> + + <div><a href="/news/spark-0-7-3-released.html">Spark 0.7.3 released</a> <span class="post-info">(July 16, 2013)</span></div> + + <div><a href="/news/spark-featured-in-wired.html">Spark featured in Wired</a> <span class="post-info">(June 21, 2013)</span></div> + + </div> + +<div style="text-align:right"><a href="/news/index.html">News Archive</a></div> + +<p><!-- Not porting the following to Pygments since it becomes a lot less colorful --></p> + +<div class="code" style="margin-top: 20px;"> + file = spark.textFile(<span class="string">"hdfs://..."</span>)<br /> + <br /> + file.<span class="sparkop">flatMap</span>(<span class="closure">line => line.split(" ")</span>)<br /> + .<span class="sparkop">map</span>(<span class="closure">word => (word, 1)</span>)<br /> + .<span class="sparkop">reduceByKey</span>(<span class="closure">_ + _</span>) + </div> +<div class="caption">Word Count implemented in Spark</div> + +<div align="center" style="margin-top: 20px;"> + <img src="/images/spark-lr.png" alt="Logistic regression performance in Spark vs Hadoop" /> + </div> +<div class="caption">Logistic regression in Spark vs Hadoop</div> +<h2 style="text-align:center"><a href="/downloads"><img src="/images/download.png" alt="Download" style="vertical-align: middle" /> Download Spark</a></h2> + + </div> + + <footer id="colophon" role="contentinfo"> + <div id="site-generator"> + <p>Spark is an open source project developed at the UC Berkeley <a href="https://amplab.cs.berkeley.edu">AMPLab</a>.</p> + <a class="amp-logo" style="background:url(/images/amplab-small.png)" href="https://amplab.cs.berkeley.edu/" title="Brought to you by the UC Berkeley AMPLab." rel="generator"><!--Brought to you by the UC Berkeley AMPLab--> </a> + </div> +</footer><!-- #colophon --> + + </div><!-- #primary --> + </div><!-- #main --> +</div><!-- #page --> + + +</body> +</html> diff --git a/talks/overview.pdf b/talks/overview.pdf Binary files differnew file mode 100644 index 000000000..df00f016d --- /dev/null +++ b/talks/overview.pdf diff --git a/talks/overview.pptx b/talks/overview.pptx Binary files differnew file mode 100644 index 000000000..26ed4e90b --- /dev/null +++ b/talks/overview.pptx diff --git a/talks/strata_spark_streaming.pdf b/talks/strata_spark_streaming.pdf Binary files differnew file mode 100644 index 000000000..4fa980efd --- /dev/null +++ b/talks/strata_spark_streaming.pdf diff --git a/talks/strata_spark_streaming.ppt b/talks/strata_spark_streaming.ppt Binary files differnew file mode 100644 index 000000000..30dfdf4af --- /dev/null +++ b/talks/strata_spark_streaming.ppt |
