core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

package com.rockymadden.stringmetric

object Tokenize {
	trait Tokenizer[A] {
		def tokenize(a: A): Option[Array[A]]
	}


	trait StringTokenizer extends Tokenizer[Array[Char]] {
		def tokenize(a: String): Option[Array[String]]
	}


	object StringTokenizer {
		val NGram = tokenize.NGramTokenizer

		def tokenizeWithNGram(n: Int)(charArray: Array[Char]) = NGram(n).tokenize(charArray)
	}
}