1 files changed, 19 insertions, 0 deletions
diff --git a/core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala b/core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala
new file mode 100755
index 0000000..0bae29a
--- /dev/null
+++ b/core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala
@@ -0,0 +1,19 @@
+package com.rockymadden.stringmetric
+
+object Tokenize {
+	trait Tokenizer[A] {
+		def tokenize(a: A): Option[Array[A]]
+	}
+
+
+	trait StringTokenizer extends Tokenizer[Array[Char]] {
+		def tokenize(a: String): Option[Array[String]]
+	}
+
+
+	object StringTokenizer {
+		val NGram = com.rockymadden.stringmetric.tokenize.NGramTokenizer
+
+		def tokenizeWithNGram(n: Int)(charArray: Array[Char]) = NGram(n).tokenize(charArray)
+	}
+}