diff options
author | Anant <anant.asty@gmail.com> | 2014-10-31 18:33:19 -0700 |
---|---|---|
committer | Xiangrui Meng <meng@databricks.com> | 2014-10-31 18:33:19 -0700 |
commit | e07fb6a41ee949f8dba44d5a3b6c0615f27f0eaf (patch) | |
tree | 761e5c6feb0a3281ca31bacb63a7d2b234828212 /examples/src/main/python/mllib/word2vec.py | |
parent | 62d01d255c001a6d397cc166a10aba3894f43459 (diff) | |
download | spark-e07fb6a41ee949f8dba44d5a3b6c0615f27f0eaf.tar.gz spark-e07fb6a41ee949f8dba44d5a3b6c0615f27f0eaf.tar.bz2 spark-e07fb6a41ee949f8dba44d5a3b6c0615f27f0eaf.zip |
[SPARK-3838][examples][mllib][python] Word2Vec example in python
This pull request refers to issue: https://issues.apache.org/jira/browse/SPARK-3838
Python example for word2vec
mengxr
Author: Anant <anant.asty@gmail.com>
Closes #2952 from anantasty/SPARK-3838 and squashes the following commits:
87bd723 [Anant] remove stop line
4bd439e [Anant] Changes as per code review. Fized error in word2vec python example, simplified example in docs.
3d3c9ee [Anant] Added empty line after python imports
0c90c31 [Anant] Fixed erroneous code. I was still treating each line to be a single word instead of 16 words
ee4f5f6 [Anant] Fixes from code review comments
c637bcf [Anant] Added word2vec python example to docs
269f31f [Anant] added example in docs
c015b14 [Anant] Added python example for word2vec
Diffstat (limited to 'examples/src/main/python/mllib/word2vec.py')
-rw-r--r-- | examples/src/main/python/mllib/word2vec.py | 50 |
1 files changed, 50 insertions, 0 deletions
diff --git a/examples/src/main/python/mllib/word2vec.py b/examples/src/main/python/mllib/word2vec.py new file mode 100644 index 0000000000..99fef4276a --- /dev/null +++ b/examples/src/main/python/mllib/word2vec.py @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This example uses text8 file from http://mattmahoney.net/dc/text8.zip +# The file was downloadded, unziped and split into multiple lines using +# +# wget http://mattmahoney.net/dc/text8.zip +# unzip text8.zip +# grep -o -E '\w+(\W+\w+){0,15}' text8 > text8_lines +# This was done so that the example can be run in local mode + + +import sys + +from pyspark import SparkContext +from pyspark.mllib.feature import Word2Vec + +USAGE = ("bin/spark-submit --driver-memory 4g " + "examples/src/main/python/mllib/word2vec.py text8_lines") + +if __name__ == "__main__": + if len(sys.argv) < 2: + print USAGE + sys.exit("Argument for file not provided") + file_path = sys.argv[1] + sc = SparkContext(appName='Word2Vec') + inp = sc.textFile(file_path).map(lambda row: row.split(" ")) + + word2vec = Word2Vec() + model = word2vec.fit(inp) + + synonyms = model.findSynonyms('china', 40) + + for word, cosine_distance in synonyms: + print "{}: {}".format(word, cosine_distance) + sc.stop() |