1 files changed, 5 insertions, 0 deletions
diff --git a/core/src/main/scala/com/rockymadden/stringmetric/Tokenizer.scala b/core/src/main/scala/com/rockymadden/stringmetric/Tokenizer.scala
new file mode 100755
index 0000000..7a2544c
--- /dev/null
+++ b/core/src/main/scala/com/rockymadden/stringmetric/Tokenizer.scala
@@ -0,0 +1,5 @@
+package com.rockymadden.stringmetric
+
+trait Tokenizer[A] {
+	def tokenize(a: A): Option[Array[A]]
+}