summaryrefslogtreecommitdiff
path: root/core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala
diff options
context:
space:
mode:
authorRocky Madden <git@rockymadden.com>2014-01-02 13:47:43 -0700
committerRocky Madden <git@rockymadden.com>2014-01-02 13:47:43 -0700
commit49de854bb464f1be37fbb27f942b9b65e52df751 (patch)
tree6c9a27ac1264648f67eba9c8707fa87d3dc5b3cd /core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala
parent42b990a1523a68717afcbdbc2cc4968c041451ec (diff)
downloadstringmetric-49de854bb464f1be37fbb27f942b9b65e52df751.tar.gz
stringmetric-49de854bb464f1be37fbb27f942b9b65e52df751.tar.bz2
stringmetric-49de854bb464f1be37fbb27f942b9b65e52df751.zip
Moved from gradle to sbt.
Diffstat (limited to 'core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala')
-rwxr-xr-xcore/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala33
1 files changed, 0 insertions, 33 deletions
diff --git a/core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala b/core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala
deleted file mode 100755
index a011c96..0000000
--- a/core/source/main/scala/com/rockymadden/stringmetric/Tokenize.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-package com.rockymadden.stringmetric
-
-object Tokenize {
- sealed trait Tokenizer[A] {
- def tokenize(a: A): Option[Array[A]]
- }
-
-
- sealed trait StringTokenizer extends Tokenizer[Array[Char]] {
- def tokenize(a: String): Option[Array[String]]
- }
-
-
- object StringTokenizer {
- val NGram = NGramTokenizer
-
- def tokenizeWithNGram(n: Int)(charArray: Array[Char]) = NGram(n).tokenize(charArray)
- }
-
-
- final case class NGramTokenizer(n: Int) extends StringTokenizer {
- override def tokenize(a: Array[Char]): Option[Array[Array[Char]]] =
- if (n <= 0 || a.length < n) None
- else Some(sequence(a, Array.empty[Array[Char]], n))
-
- override def tokenize(a: String): Option[Array[String]] = tokenize(a.toCharArray).map(_.map(_.mkString))
-
- @annotation.tailrec
- private val sequence: ((Array[Char], Array[Array[Char]], Int) => Array[Array[Char]]) = (i, o, n) =>
- if (i.length <= n) o :+ i
- else sequence(i.tail, o :+ i.take(n), n)
- }
-}