diff options
author | Rocky Madden <git@rockymadden.com> | 2014-01-02 13:47:43 -0700 |
---|---|---|
committer | Rocky Madden <git@rockymadden.com> | 2014-01-02 13:47:43 -0700 |
commit | 49de854bb464f1be37fbb27f942b9b65e52df751 (patch) | |
tree | 6c9a27ac1264648f67eba9c8707fa87d3dc5b3cd /core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala | |
parent | 42b990a1523a68717afcbdbc2cc4968c041451ec (diff) | |
download | stringmetric-49de854bb464f1be37fbb27f942b9b65e52df751.tar.gz stringmetric-49de854bb464f1be37fbb27f942b9b65e52df751.tar.bz2 stringmetric-49de854bb464f1be37fbb27f942b9b65e52df751.zip |
Moved from gradle to sbt.
Diffstat (limited to 'core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala')
-rwxr-xr-x | core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala | 33 |
1 files changed, 0 insertions, 33 deletions
diff --git a/core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala b/core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala deleted file mode 100755 index a011c96..0000000 --- a/core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.rockymadden.stringmetric - -object Tokenize { - sealed trait Tokenizer[A] { - def tokenize(a: A): Option[Array[A]] - } - - - sealed trait StringTokenizer extends Tokenizer[Array[Char]] { - def tokenize(a: String): Option[Array[String]] - } - - - object StringTokenizer { - val NGram = NGramTokenizer - - def tokenizeWithNGram(n: Int)(charArray: Array[Char]) = NGram(n).tokenize(charArray) - } - - - final case class NGramTokenizer(n: Int) extends StringTokenizer { - override def tokenize(a: Array[Char]): Option[Array[Array[Char]]] = - if (n <= 0 || a.length < n) None - else Some(sequence(a, Array.empty[Array[Char]], n)) - - override def tokenize(a: String): Option[Array[String]] = tokenize(a.toCharArray).map(_.map(_.mkString)) - - @annotation.tailrec - private val sequence: ((Array[Char], Array[Array[Char]], Int) => Array[Array[Char]]) = (i, o, n) => - if (i.length <= n) o :+ i - else sequence(i.tail, o :+ i.take(n), n) - } -} |