aboutsummaryrefslogtreecommitdiff
path: root/sql/core/src/test/java
diff options
context:
space:
mode:
Diffstat (limited to 'sql/core/src/test/java')
-rw-r--r--sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java33
1 files changed, 25 insertions, 8 deletions
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
index 10ee7d57c7..1eb680dc4c 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
@@ -18,6 +18,8 @@
package test.org.apache.spark.sql;
import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.net.URL;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
@@ -303,27 +305,42 @@ public class JavaDataFrameSuite {
Assert.assertEquals(30000.0, actual.get(1).getDouble(2), 0.01);
}
+ private String getResource(String resource) {
+ try {
+ // The following "getResource" has different behaviors in SBT and Maven.
+ // When running in Jenkins, the file path may contain "@" when there are multiple
+ // SparkPullRequestBuilders running in the same worker
+ // (e.g., /home/jenkins/workspace/SparkPullRequestBuilder@2)
+ // When running in SBT, "@" in the file path will be returned as "@", however,
+ // when running in Maven, "@" will be encoded as "%40".
+ // Therefore, we convert it to URI then call "getPath" to decode it back so that it can both
+ // work both in SBT and Maven.
+ URL url = Thread.currentThread().getContextClassLoader().getResource(resource);
+ return url.toURI().getPath();
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
@Test
public void testGenericLoad() {
- Dataset<Row> df1 = context.read().format("text").load(
- Thread.currentThread().getContextClassLoader().getResource("text-suite.txt").toString());
+ Dataset<Row> df1 = context.read().format("text").load(getResource("text-suite.txt"));
Assert.assertEquals(4L, df1.count());
Dataset<Row> df2 = context.read().format("text").load(
- Thread.currentThread().getContextClassLoader().getResource("text-suite.txt").toString(),
- Thread.currentThread().getContextClassLoader().getResource("text-suite2.txt").toString());
+ getResource("text-suite.txt"),
+ getResource("text-suite2.txt"));
Assert.assertEquals(5L, df2.count());
}
@Test
public void testTextLoad() {
- Dataset<String> ds1 = context.read().text(
- Thread.currentThread().getContextClassLoader().getResource("text-suite.txt").toString());
+ Dataset<String> ds1 = context.read().text(getResource("text-suite.txt"));
Assert.assertEquals(4L, ds1.count());
Dataset<String> ds2 = context.read().text(
- Thread.currentThread().getContextClassLoader().getResource("text-suite.txt").toString(),
- Thread.currentThread().getContextClassLoader().getResource("text-suite2.txt").toString());
+ getResource("text-suite.txt"),
+ getResource("text-suite2.txt"));
Assert.assertEquals(5L, ds2.count());
}