blob: bcb405ba72c4ce891b15763f8d616033cd594de8 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
import glob
import os
from py4j.java_gateway import java_import, JavaGateway
SPARK_HOME = os.environ["SPARK_HOME"]
assembly_jar = glob.glob(os.path.join(SPARK_HOME, "core/target") + \
"/spark-core-assembly-*-SNAPSHOT.jar")[0]
def launch_gateway():
gateway = JavaGateway.launch_gateway(classpath=assembly_jar,
javaopts=["-Xmx256m"], die_on_exit=True)
java_import(gateway.jvm, "spark.api.java.*")
java_import(gateway.jvm, "spark.api.python.*")
java_import(gateway.jvm, "scala.Tuple2")
return gateway
|
