aboutsummaryrefslogtreecommitdiff
path: root/examples/src/main/python
diff options
context:
space:
mode:
authorDongjoon Hyun <dongjoon@apache.org>2016-06-10 15:40:29 -0700
committerReynold Xin <rxin@databricks.com>2016-06-10 15:40:29 -0700
commit2022afe57dbf8cb0c9909399962c4a3649e0601c (patch)
tree3f850e4ad28effb207044df759aade5b24afd394 /examples/src/main/python
parent127a6678d7af6b5164a115be7c64525bb80001fe (diff)
downloadspark-2022afe57dbf8cb0c9909399962c4a3649e0601c.tar.gz
spark-2022afe57dbf8cb0c9909399962c4a3649e0601c.tar.bz2
spark-2022afe57dbf8cb0c9909399962c4a3649e0601c.zip
[SPARK-15773][CORE][EXAMPLE] Avoid creating local variable `sc` in examples if possible
## What changes were proposed in this pull request? Instead of using local variable `sc` like the following example, this PR uses `spark.sparkContext`. This makes examples more concise, and also fixes some misleading, i.e., creating SparkContext from SparkSession. ``` - println("Creating SparkContext") - val sc = spark.sparkContext - println("Writing local file to DFS") val dfsFilename = dfsDirPath + "/dfs_read_write_test" - val fileRDD = sc.parallelize(fileContents) + val fileRDD = spark.sparkContext.parallelize(fileContents) ``` This will change 12 files (+30 lines, -52 lines). ## How was this patch tested? Manual. Author: Dongjoon Hyun <dongjoon@apache.org> Closes #13520 from dongjoon-hyun/SPARK-15773.
Diffstat (limited to 'examples/src/main/python')
-rwxr-xr-xexamples/src/main/python/pi.py4
-rwxr-xr-xexamples/src/main/python/transitive_closure.py4
2 files changed, 2 insertions, 6 deletions
diff --git a/examples/src/main/python/pi.py b/examples/src/main/python/pi.py
index b39d710540..e3f0c4aeef 100755
--- a/examples/src/main/python/pi.py
+++ b/examples/src/main/python/pi.py
@@ -32,8 +32,6 @@ if __name__ == "__main__":
.appName("PythonPi")\
.getOrCreate()
- sc = spark.sparkContext
-
partitions = int(sys.argv[1]) if len(sys.argv) > 1 else 2
n = 100000 * partitions
@@ -42,7 +40,7 @@ if __name__ == "__main__":
y = random() * 2 - 1
return 1 if x ** 2 + y ** 2 < 1 else 0
- count = sc.parallelize(range(1, n + 1), partitions).map(f).reduce(add)
+ count = spark.sparkContext.parallelize(range(1, n + 1), partitions).map(f).reduce(add)
print("Pi is roughly %f" % (4.0 * count / n))
spark.stop()
diff --git a/examples/src/main/python/transitive_closure.py b/examples/src/main/python/transitive_closure.py
index d88ea94e41..49551d4085 100755
--- a/examples/src/main/python/transitive_closure.py
+++ b/examples/src/main/python/transitive_closure.py
@@ -46,10 +46,8 @@ if __name__ == "__main__":
.appName("PythonTransitiveClosure")\
.getOrCreate()
- sc = spark.sparkContext
-
partitions = int(sys.argv[1]) if len(sys.argv) > 1 else 2
- tc = sc.parallelize(generateGraph(), partitions).cache()
+ tc = spark.sparkContext.parallelize(generateGraph(), partitions).cache()
# Linear transitive closure: each round grows paths by one edge,
# by joining the graph's edges with the already-discovered paths.