diff options
author | Punya Biswal <pbiswal@palantir.com> | 2014-02-22 17:53:48 -0800 |
---|---|---|
committer | Aaron Davidson <aaron@databricks.com> | 2014-02-22 17:53:48 -0800 |
commit | 29ac7ea52fbb0c6531e14305e2fb1ccba9678f7e (patch) | |
tree | 01349562ee684de12d8b0207a4548a8ddd30d385 /core/src/main/java/org | |
parent | 1aa4f8af7220bc311196ef0eef0a4814cd2757d3 (diff) | |
download | spark-29ac7ea52fbb0c6531e14305e2fb1ccba9678f7e.tar.gz spark-29ac7ea52fbb0c6531e14305e2fb1ccba9678f7e.tar.bz2 spark-29ac7ea52fbb0c6531e14305e2fb1ccba9678f7e.zip |
Migrate Java code to Scala or move it to src/main/java
These classes can't be migrated:
StorageLevels: impossible to create static fields in Scala
JavaSparkContextVarargsWorkaround: incompatible varargs
JavaAPISuite: should test Java APIs in pure Java (for sanity)
Author: Punya Biswal <pbiswal@palantir.com>
Closes #605 from punya/move-java-sources and squashes the following commits:
25b00b2 [Punya Biswal] Remove redundant type param; reformat
853da46 [Punya Biswal] Use factory method rather than constructor
e5d53d9 [Punya Biswal] Migrate Java code to Scala or move it to src/main/java
Diffstat (limited to 'core/src/main/java/org')
-rw-r--r-- | core/src/main/java/org/apache/spark/api/java/JavaSparkContextVarargsWorkaround.java | 63 | ||||
-rw-r--r-- | core/src/main/java/org/apache/spark/api/java/StorageLevels.java | 48 |
2 files changed, 111 insertions, 0 deletions
diff --git a/core/src/main/java/org/apache/spark/api/java/JavaSparkContextVarargsWorkaround.java b/core/src/main/java/org/apache/spark/api/java/JavaSparkContextVarargsWorkaround.java new file mode 100644 index 0000000000..2090efd3b9 --- /dev/null +++ b/core/src/main/java/org/apache/spark/api/java/JavaSparkContextVarargsWorkaround.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.api.java; + +import java.util.ArrayList; +import java.util.List; + +// See +// http://scala-programming-language.1934581.n4.nabble.com/Workaround-for-implementing-java-varargs-in-2-7-2-final-tp1944767p1944772.html +abstract class JavaSparkContextVarargsWorkaround { + public <T> JavaRDD<T> union(JavaRDD<T>... rdds) { + if (rdds.length == 0) { + throw new IllegalArgumentException("Union called on empty list"); + } + ArrayList<JavaRDD<T>> rest = new ArrayList<JavaRDD<T>>(rdds.length - 1); + for (int i = 1; i < rdds.length; i++) { + rest.add(rdds[i]); + } + return union(rdds[0], rest); + } + + public JavaDoubleRDD union(JavaDoubleRDD... rdds) { + if (rdds.length == 0) { + throw new IllegalArgumentException("Union called on empty list"); + } + ArrayList<JavaDoubleRDD> rest = new ArrayList<JavaDoubleRDD>(rdds.length - 1); + for (int i = 1; i < rdds.length; i++) { + rest.add(rdds[i]); + } + return union(rdds[0], rest); + } + + public <K, V> JavaPairRDD<K, V> union(JavaPairRDD<K, V>... rdds) { + if (rdds.length == 0) { + throw new IllegalArgumentException("Union called on empty list"); + } + ArrayList<JavaPairRDD<K, V>> rest = new ArrayList<JavaPairRDD<K, V>>(rdds.length - 1); + for (int i = 1; i < rdds.length; i++) { + rest.add(rdds[i]); + } + return union(rdds[0], rest); + } + + // These methods take separate "first" and "rest" elements to avoid having the same type erasure + abstract public <T> JavaRDD<T> union(JavaRDD<T> first, List<JavaRDD<T>> rest); + abstract public JavaDoubleRDD union(JavaDoubleRDD first, List<JavaDoubleRDD> rest); + abstract public <K, V> JavaPairRDD<K, V> union(JavaPairRDD<K, V> first, List<JavaPairRDD<K, V>> rest); +} diff --git a/core/src/main/java/org/apache/spark/api/java/StorageLevels.java b/core/src/main/java/org/apache/spark/api/java/StorageLevels.java new file mode 100644 index 0000000000..9f13b39909 --- /dev/null +++ b/core/src/main/java/org/apache/spark/api/java/StorageLevels.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.api.java; + +import org.apache.spark.storage.StorageLevel; + +/** + * Expose some commonly useful storage level constants. + */ +public class StorageLevels { + public static final StorageLevel NONE = create(false, false, false, 1); + public static final StorageLevel DISK_ONLY = create(true, false, false, 1); + public static final StorageLevel DISK_ONLY_2 = create(true, false, false, 2); + public static final StorageLevel MEMORY_ONLY = create(false, true, true, 1); + public static final StorageLevel MEMORY_ONLY_2 = create(false, true, true, 2); + public static final StorageLevel MEMORY_ONLY_SER = create(false, true, false, 1); + public static final StorageLevel MEMORY_ONLY_SER_2 = create(false, true, false, 2); + public static final StorageLevel MEMORY_AND_DISK = create(true, true, true, 1); + public static final StorageLevel MEMORY_AND_DISK_2 = create(true, true, true, 2); + public static final StorageLevel MEMORY_AND_DISK_SER = create(true, true, false, 1); + public static final StorageLevel MEMORY_AND_DISK_SER_2 = create(true, true, false, 2); + + /** + * Create a new StorageLevel object. + * @param useDisk saved to disk, if true + * @param useMemory saved to memory, if true + * @param deserialized saved as deserialized objects, if true + * @param replication replication factor + */ + public static StorageLevel create(boolean useDisk, boolean useMemory, boolean deserialized, int replication) { + return StorageLevel.apply(useDisk, useMemory, deserialized, replication); + } +} |