aboutsummaryrefslogtreecommitdiff
path: root/python/pyspark
diff options
context:
space:
mode:
authorJosh Rosen <joshrosen@eecs.berkeley.edu>2013-01-23 11:18:25 -0800
committerJosh Rosen <joshrosen@eecs.berkeley.edu>2013-01-23 11:47:27 -0800
commitb47d054cfc5ef45b92a1c970388722ffa0283e66 (patch)
tree94e836802d72085680be3a13758e269b9eb9a7b9 /python/pyspark
parentc75ae3622eeed068c44b1f823ef4d87d01a720fd (diff)
downloadspark-b47d054cfc5ef45b92a1c970388722ffa0283e66.tar.gz
spark-b47d054cfc5ef45b92a1c970388722ffa0283e66.tar.bz2
spark-b47d054cfc5ef45b92a1c970388722ffa0283e66.zip
Remove use of abc.ABCMeta due to cloudpickle issue.
cloudpickle runs into issues while pickling subclasses of AccumulatorParam, which may be related to this Python issue: http://bugs.python.org/issue7689 This seems hard to fix and the ABCMeta wasn't necessary, so I removed it.
Diffstat (limited to 'python/pyspark')
-rw-r--r--python/pyspark/accumulators.py11
1 files changed, 4 insertions, 7 deletions
diff --git a/python/pyspark/accumulators.py b/python/pyspark/accumulators.py
index 5a9269f9bb..61fcbbd376 100644
--- a/python/pyspark/accumulators.py
+++ b/python/pyspark/accumulators.py
@@ -25,7 +25,8 @@
>>> a.value
13
->>> class VectorAccumulatorParam(object):
+>>> from pyspark.accumulators import AccumulatorParam
+>>> class VectorAccumulatorParam(AccumulatorParam):
... def zero(self, value):
... return [0.0] * len(value)
... def addInPlace(self, val1, val2):
@@ -61,7 +62,6 @@ Traceback (most recent call last):
Exception:...
"""
-from abc import ABCMeta, abstractmethod
import struct
import SocketServer
import threading
@@ -138,23 +138,20 @@ class AccumulatorParam(object):
"""
Helper object that defines how to accumulate values of a given type.
"""
- __metaclass__ = ABCMeta
- @abstractmethod
def zero(self, value):
"""
Provide a "zero value" for the type, compatible in dimensions with the
provided C{value} (e.g., a zero vector)
"""
- return
+ raise NotImplementedError
- @abstractmethod
def addInPlace(self, value1, value2):
"""
Add two values of the accumulator's data type, returning a new value;
for efficiency, can also update C{value1} in place and return it.
"""
- return
+ raise NotImplementedError
class AddingAccumulatorParam(AccumulatorParam):