1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
|
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.spark.sql.types
import scala.reflect.runtime.universe.TypeTag
import org.apache.spark.annotation.InterfaceStability
import org.apache.spark.sql.catalyst.expressions.Expression
/**
* A non-concrete data type, reserved for internal uses.
*/
private[sql] abstract class AbstractDataType {
/**
* The default concrete type to use if we want to cast a null literal into this type.
*/
private[sql] def defaultConcreteType: DataType
/**
* Returns true if `other` is an acceptable input type for a function that expects this,
* possibly abstract DataType.
*
* {{{
* // this should return true
* DecimalType.acceptsType(DecimalType(10, 2))
*
* // this should return true as well
* NumericType.acceptsType(DecimalType(10, 2))
* }}}
*/
private[sql] def acceptsType(other: DataType): Boolean
/** Readable string representation for the type. */
private[sql] def simpleString: String
}
/**
* A collection of types that can be used to specify type constraints. The sequence also specifies
* precedence: an earlier type takes precedence over a latter type.
*
* {{{
* TypeCollection(StringType, BinaryType)
* }}}
*
* This means that we prefer StringType over BinaryType if it is possible to cast to StringType.
*/
private[sql] class TypeCollection(private val types: Seq[AbstractDataType])
extends AbstractDataType {
require(types.nonEmpty, s"TypeCollection ($types) cannot be empty")
override private[sql] def defaultConcreteType: DataType = types.head.defaultConcreteType
override private[sql] def acceptsType(other: DataType): Boolean =
types.exists(_.acceptsType(other))
override private[sql] def simpleString: String = {
types.map(_.simpleString).mkString("(", " or ", ")")
}
}
private[sql] object TypeCollection {
/**
* Types that can be ordered/compared. In the long run we should probably make this a trait
* that can be mixed into each data type, and perhaps create an [[AbstractDataType]].
*/
// TODO: Should we consolidate this with RowOrdering.isOrderable?
val Ordered = TypeCollection(
BooleanType,
ByteType, ShortType, IntegerType, LongType,
FloatType, DoubleType, DecimalType,
TimestampType, DateType,
StringType, BinaryType)
/**
* Types that include numeric types and interval type. They are only used in unary_minus,
* unary_positive, add and subtract operations.
*/
val NumericAndInterval = TypeCollection(NumericType, CalendarIntervalType)
def apply(types: AbstractDataType*): TypeCollection = new TypeCollection(types)
def unapply(typ: AbstractDataType): Option[Seq[AbstractDataType]] = typ match {
case typ: TypeCollection => Some(typ.types)
case _ => None
}
}
/**
* An [[AbstractDataType]] that matches any concrete data types.
*/
protected[sql] object AnyDataType extends AbstractDataType {
// Note that since AnyDataType matches any concrete types, defaultConcreteType should never
// be invoked.
override private[sql] def defaultConcreteType: DataType = throw new UnsupportedOperationException
override private[sql] def simpleString: String = "any"
override private[sql] def acceptsType(other: DataType): Boolean = true
}
/**
* An internal type used to represent everything that is not null, UDTs, arrays, structs, and maps.
*/
protected[sql] abstract class AtomicType extends DataType {
private[sql] type InternalType
private[sql] val tag: TypeTag[InternalType]
private[sql] val ordering: Ordering[InternalType]
}
object AtomicType {
/**
* Enables matching against AtomicType for expressions:
* {{{
* case Cast(child @ AtomicType(), StringType) =>
* ...
* }}}
*/
def unapply(e: Expression): Boolean = e.dataType.isInstanceOf[AtomicType]
}
/**
* Numeric data types.
*
* @since 1.3.0
*/
@InterfaceStability.Stable
abstract class NumericType extends AtomicType {
// Unfortunately we can't get this implicitly as that breaks Spark Serialization. In order for
// implicitly[Numeric[JvmType]] to be valid, we have to change JvmType from a type variable to a
// type parameter and add a numeric annotation (i.e., [JvmType : Numeric]). This gets
// desugared by the compiler into an argument to the objects constructor. This means there is no
// longer a no argument constructor and thus the JVM cannot serialize the object anymore.
private[sql] val numeric: Numeric[InternalType]
}
private[sql] object NumericType extends AbstractDataType {
/**
* Enables matching against NumericType for expressions:
* {{{
* case Cast(child @ NumericType(), StringType) =>
* ...
* }}}
*/
def unapply(e: Expression): Boolean = e.dataType.isInstanceOf[NumericType]
override private[sql] def defaultConcreteType: DataType = DoubleType
override private[sql] def simpleString: String = "numeric"
override private[sql] def acceptsType(other: DataType): Boolean = other.isInstanceOf[NumericType]
}
private[sql] object IntegralType extends AbstractDataType {
/**
* Enables matching against IntegralType for expressions:
* {{{
* case Cast(child @ IntegralType(), StringType) =>
* ...
* }}}
*/
def unapply(e: Expression): Boolean = e.dataType.isInstanceOf[IntegralType]
override private[sql] def defaultConcreteType: DataType = IntegerType
override private[sql] def simpleString: String = "integral"
override private[sql] def acceptsType(other: DataType): Boolean = other.isInstanceOf[IntegralType]
}
private[sql] abstract class IntegralType extends NumericType {
private[sql] val integral: Integral[InternalType]
}
private[sql] object FractionalType {
/**
* Enables matching against FractionalType for expressions:
* {{{
* case Cast(child @ FractionalType(), StringType) =>
* ...
* }}}
*/
def unapply(e: Expression): Boolean = e.dataType.isInstanceOf[FractionalType]
}
private[sql] abstract class FractionalType extends NumericType {
private[sql] val fractional: Fractional[InternalType]
private[sql] val asIntegral: Integral[InternalType]
}
|
