blob: 1ea9ba55412849189870e01938d5a529848489ed (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
|
#!/usr/bin/env bash
# This file contains a few additional setting that are useful for
# running streaming jobs in Spark. Copy this file as streaming-env.sh .
# Note that this shell script will be read after spark-env.sh, so settings
# in this file may override similar settings (if present) in spark-env.sh .
# Using concurrent GC is strongly recommended as it can significantly
# reduce GC related pauses.
SPARK_JAVA_OPTS+=" -XX:+UseConcMarkSweepGC"
# Using Kryo serialization can improve serialization performance
# and therefore the throughput of the Spark Streaming programs. However,
# using Kryo serialization with custom classes may required you to
# register the classes with Kryo. Refer to the Spark documentation
# for more details.
# SPARK_JAVA_OPTS+=" -Dspark.serializer=spark.KryoSerializer"
export SPARK_JAVA_OPTS
|
