diff options
Diffstat (limited to 'core/src/main')
26 files changed, 1159 insertions, 0 deletions
diff --git a/core/src/main/scala/com/rockymadden/stringmetric/Algorithm.scala b/core/src/main/scala/com/rockymadden/stringmetric/Algorithm.scala new file mode 100755 index 0000000..84f136d --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/Algorithm.scala @@ -0,0 +1,46 @@ +package com.rockymadden.stringmetric + +object Algorithm { + import Transform.StringTransform + + + trait Algorithm[A] { + def compute(a: A): Option[A] + } + + + trait StringAlgorithm extends Algorithm[Array[Char]] { + def compute(a: String): Option[String] + } + + + object StringAlgorithm { + final val Metaphone = phonetic.MetaphoneAlgorithm + final val Nysiis = phonetic.NysiisAlgorithm + final val RefinedNysiis = phonetic.RefinedNysiisAlgorithm + final val RefinedSoundex = phonetic.RefinedSoundexAlgorithm + final val Soundex = phonetic.SoundexAlgorithm + + def computeWithMetaphone(a: Array[Char]) = Metaphone.compute(a) + + def computeWithNysiis(a: Array[Char]) = Nysiis.compute(a) + + def computeWithRefinedNysiis(a: Array[Char]) = RefinedNysiis.compute(a) + + def computeWithRefinedSoundex(a: Array[Char]) = RefinedSoundex.compute(a) + + def computeWithSoundex(a: Array[Char]) = Soundex.compute(a) + } + + + final class StringAlgorithmDecorator(val sa: StringAlgorithm) { + val withTransform: (StringTransform => StringAlgorithm) = (st) => new StringAlgorithm { + private[this] val self: StringAlgorithm = sa + private[this] val transform: StringTransform = st + + override def compute(a: Array[Char]): Option[Array[Char]] = self.compute(transform(a)) + + override def compute(a: String): Option[String] = self.compute(transform(a.toCharArray)).map(_.mkString) + } + } +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/Alphabet.scala b/core/src/main/scala/com/rockymadden/stringmetric/Alphabet.scala new file mode 100755 index 0000000..5e666d2 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/Alphabet.scala @@ -0,0 +1,42 @@ +package com.rockymadden.stringmetric + +import scala.collection.immutable.Set + +object Alphabet { + sealed abstract class AlphabetSet(val chars: Set[Char]) { + def isSuperset(a: Char): Boolean = chars.contains(a) + + def isSuperset(a: Array[Char]): Boolean = a.length > 0 && a.takeWhile(chars.contains).length == a.length + + def isSuperset(a: String): Boolean = isSuperset(a.toCharArray) + } + + + case object LowercaseConsonant extends AlphabetSet( + Set('b', 'c', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'm', 'n', 'p', 'q', 'r', 's', 't', 'v', 'w', 'x' ,'z') + ) + + case object UppercaseConsonant extends AlphabetSet( + Set('B', 'C', 'D', 'F', 'G', 'H', 'J', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'V', 'W', 'X' ,'Z') + ) + + case object Consonant extends AlphabetSet(LowercaseConsonant.chars ++ UppercaseConsonant.chars) + + case object LowercaseVowel extends AlphabetSet(Set('a', 'e', 'i', 'o', 'u')) + + case object UppercaseVowel extends AlphabetSet(Set('A', 'E', 'I', 'O', 'U')) + + case object Vowel extends AlphabetSet(LowercaseVowel.chars ++ UppercaseVowel.chars) + + case object LowercaseY extends AlphabetSet(Set('y')) + + case object UppercaseY extends AlphabetSet(Set('Y')) + + case object Y extends AlphabetSet(LowercaseY.chars ++ UppercaseY.chars) + + case object LowercaseAlpha extends AlphabetSet(LowercaseConsonant.chars ++ LowercaseVowel.chars ++ LowercaseY.chars) + + case object UppercaseAlpha extends AlphabetSet(UppercaseConsonant.chars ++ UppercaseVowel.chars ++ UppercaseY.chars) + + case object Alpha extends AlphabetSet(LowercaseAlpha.chars ++ UppercaseAlpha.chars) +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/Metric.scala b/core/src/main/scala/com/rockymadden/stringmetric/Metric.scala new file mode 100755 index 0000000..f45dd14 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/Metric.scala @@ -0,0 +1,77 @@ +package com.rockymadden.stringmetric + +object Metric { + import Transform.StringTransform + + + trait Metric[A, B] { + def compare(a: A, b: A): Option[B] + } + + + trait StringMetric[A] extends Metric[Array[Char], A] { + def compare(a: String, b: String): Option[A] + } + + + object StringMetric { + final val DiceSorensen = similarity.DiceSorensenMetric + final val Hamming = similarity.HammingMetric + final val Jaccard = similarity.JaccardMetric + final val Jaro = similarity.JaroMetric + final val JaroWinkler = similarity.JaroWinklerMetric + final val Levenshtein = similarity.LevenshteinMetric + final val Metaphone = phonetic.MetaphoneMetric + final val NGram = similarity.NGramMetric + final val Nysiis = phonetic.NysiisMetric + final val Overlap = similarity.OverlapMetric + final val RefinedNysiis = phonetic.RefinedNysiisMetric + final val RefinedSoundex = phonetic.RefinedSoundexMetric + final val Soundex = phonetic.SoundexMetric + final val WeightedLevenshtein = similarity.WeightedLevenshteinMetric + + def compareWithDiceSorensen(n: Int)(a: Array[Char], b: Array[Char]) = DiceSorensen(n).compare(a, b) + + def compareWithHamming(a: Array[Char], b: Array[Char]) = Hamming.compare(a, b) + + def compareWithJaccard(n: Int)(a: Array[Char], b: Array[Char]) = Jaccard(n).compare(a, b) + + def compareWithJaro(a: Array[Char], b: Array[Char]) = Jaro.compare(a, b) + + def compareWithJaroWinkler(a: Array[Char], b: Array[Char]) = JaroWinkler.compare(a, b) + + def compareWithLevenshtein(a: Array[Char], b: Array[Char]) = Levenshtein.compare(a, b) + + def compareWithMetaphone(a: Array[Char], b: Array[Char]) = Metaphone.compare(a, b) + + def compareWithNGram(n: Int)(a: Array[Char], b: Array[Char]) = NGram(n).compare(a, b) + + def compareWithNysiis(a: Array[Char], b: Array[Char]) = Nysiis.compare(a, b) + + def compareWithOverlap(n: Int)(a: Array[Char], b: Array[Char]) = Overlap(n).compare(a, b) + + def compareWithRefinedNysiis(a: Array[Char], b: Array[Char]) = RefinedNysiis.compare(a, b) + + def compareWithRefinedSoundex(a: Array[Char], b: Array[Char]) = RefinedSoundex.compare(a, b) + + def compareWithSoundex(a: Array[Char], b: Array[Char]) = Soundex.compare(a, b) + + def compareWithWeightedLevenshtein(delete: BigDecimal, insert: BigDecimal, substitute: BigDecimal) + (a: Array[Char], b: Array[Char]) = + + WeightedLevenshtein(delete, insert, substitute).compare(a, b) + } + + final class StringMetricDecorator[A](val sm: StringMetric[A]) { + val withTransform: (StringTransform => StringMetric[A]) = (st) => new StringMetric[A] { + private[this] val self: StringMetric[A] = sm + private[this] val transform: StringTransform = st + + override def compare(a: Array[Char], b: Array[Char]): Option[A] = + self.compare(transform(a), transform(b)) + + override def compare(a: String, b: String): Option[A] = + self.compare(transform(a.toCharArray), transform(b.toCharArray)) + } + } +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/Tokenize.scala b/core/src/main/scala/com/rockymadden/stringmetric/Tokenize.scala new file mode 100755 index 0000000..a011c96 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/Tokenize.scala @@ -0,0 +1,33 @@ +package com.rockymadden.stringmetric + +object Tokenize { + sealed trait Tokenizer[A] { + def tokenize(a: A): Option[Array[A]] + } + + + sealed trait StringTokenizer extends Tokenizer[Array[Char]] { + def tokenize(a: String): Option[Array[String]] + } + + + object StringTokenizer { + val NGram = NGramTokenizer + + def tokenizeWithNGram(n: Int)(charArray: Array[Char]) = NGram(n).tokenize(charArray) + } + + + final case class NGramTokenizer(n: Int) extends StringTokenizer { + override def tokenize(a: Array[Char]): Option[Array[Array[Char]]] = + if (n <= 0 || a.length < n) None + else Some(sequence(a, Array.empty[Array[Char]], n)) + + override def tokenize(a: String): Option[Array[String]] = tokenize(a.toCharArray).map(_.map(_.mkString)) + + @annotation.tailrec + private val sequence: ((Array[Char], Array[Array[Char]], Int) => Array[Array[Char]]) = (i, o, n) => + if (i.length <= n) o :+ i + else sequence(i.tail, o :+ i.take(n), n) + } +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/Transform.scala b/core/src/main/scala/com/rockymadden/stringmetric/Transform.scala new file mode 100644 index 0000000..c2cdace --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/Transform.scala @@ -0,0 +1,71 @@ +package com.rockymadden.stringmetric + +object Transform { + import scala.collection.immutable.NumericRange + + + type Transform[A] = (A => A) + type StringTransform = Transform[Array[Char]] + + + object StringTransform { + private final val Ascii = NumericRange(0x00, 0x7F, 1) + private final val ExtendedAscii = NumericRange(0x00, 0x7F, 1) + private final val Latin = NumericRange(0x00, 0x24F, 1) + private final val LowerCase = NumericRange(0x61, 0x7A, 1) + private final val Numbers = NumericRange(0x30, 0x39, 1) + private final val UpperCase = NumericRange(0x41, 0x5A, 1) + + private final val filter: ((Array[Char], (Char => Boolean)) => String) = (ca, f) => + ca.filter(c => f(c)).mkString + + private final val filterNot: ((Array[Char], (Char => Boolean)) => String) = (ca, f) => + ca.filterNot(c => f(c)).mkString + + val filterAlpha: StringTransform = (ca) => filter(ca, c => { + val ci = c.toInt + LowerCase.contains(ci) || UpperCase.contains(ci) + }) + + val filterNotAlpha: StringTransform = (ca) => filterNot(ca, c => { + val ci = c.toInt + LowerCase.contains(ci) || UpperCase.contains(ci) + }) + + val filterAlphaNumeric: StringTransform = (ca) => filter(ca, c => { + val ci = c.toInt + LowerCase.contains(ci) || UpperCase.contains(ci) || Numbers.contains(ci) + }) + + val filterNotAlphaNumeric: StringTransform = (ca) => filterNot(ca, c => { + val ci = c.toInt + LowerCase.contains(ci) || UpperCase.contains(ci) || Numbers.contains(ci) + }) + + val filterAscii: StringTransform = (ca) => filter(ca, c => Ascii.contains(c.toInt)) + + val filterNotAscii: StringTransform = (ca) => filterNot(ca, c => Ascii.contains(c.toInt)) + + val filterExtendedAscii: StringTransform = (ca) => filter(ca, c => ExtendedAscii.contains(c.toInt)) + + val filterNotExtendedAscii: StringTransform = (ca) => filterNot(ca, c => ExtendedAscii.contains(c.toInt)) + + val filterLatin: StringTransform = (ca) => filter(ca, c => Latin.contains(c.toInt)) + + val filterNotLatin: StringTransform = (ca) => filterNot(ca, c => Latin.contains(c.toInt)) + + val filterLowerCase: StringTransform = (ca) => filter(ca, c => LowerCase.contains(c.toInt)) + + val filterNotLowerCase: StringTransform = (ca) => filterNot(ca, c => LowerCase.contains(c.toInt)) + + val filterNumeric: StringTransform = (ca) => filter(ca, c => Numbers.contains(c.toInt)) + + val filterNotNumeric: StringTransform = (ca) => filterNot(ca, c => Numbers.contains(c.toInt)) + + val filterUpperCase: StringTransform = (ca) => filter(ca, c => UpperCase.contains(c.toInt)) + + val filterNotUpperCase: StringTransform = (ca) => filterNot(ca, c => UpperCase.contains(c.toInt)) + + val ignoreAlphaCase: StringTransform = (ca) => ca.map(c => if (c >= 65 && c <= 90) (c + 32).toChar else c) + } +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/package.scala b/core/src/main/scala/com/rockymadden/stringmetric/package.scala new file mode 100755 index 0000000..e5bc19d --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/package.scala @@ -0,0 +1,17 @@ +package com.rockymadden + +package object stringmetric { + import scala.language.implicitConversions + import Algorithm._ + import Metric._ + + type CompareTuple[T] = (Array[T], Array[T]) + type MatchTuple[T] = (Array[T], Array[T]) + + implicit def stringToCharArray(s: String): Array[Char] = + s.toCharArray + implicit def stringAlgorithmToDecoratedStringAlgorithm(sa: StringAlgorithm): StringAlgorithmDecorator = + new StringAlgorithmDecorator(sa) + implicit def stringMetricToDecoratedStringMetric[A](sa: StringMetric[A]): StringMetricDecorator[A] = + new StringMetricDecorator[A](sa) +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/phonetic/MetaphoneAlgorithm.scala b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/MetaphoneAlgorithm.scala new file mode 100755 index 0000000..3abe7cc --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/MetaphoneAlgorithm.scala @@ -0,0 +1,105 @@ +package com.rockymadden.stringmetric.phonetic + +import com.rockymadden.stringmetric.Algorithm.StringAlgorithm + +case object MetaphoneAlgorithm extends StringAlgorithm { + import com.rockymadden.stringmetric.Alphabet.{Alpha, LowercaseVowel} + + override def compute(a: Array[Char]): Option[Array[Char]] = + if (a.length == 0 || !(Alpha isSuperset a.head)) None + else { + val th = (transcodeHead andThen deduplicate)(a.map(_.toLower)) + val t = transcode(Array.empty[Char], th.head, th.tail, Array.empty[Char]) + + if (t.length == 0) None else Some(t) // Single Y or W would have 0 length. + } + + override def compute(a: String): Option[String] = compute(a.toCharArray).map(_.mkString) + + private val deduplicate: (Array[Char] => Array[Char]) = (ca) => + if (ca.length <= 1) ca + else ca.sliding(2).withFilter(a => a(0) == 'c' || a(0) != a(1)).map(_(0)).toArray[Char] :+ ca.last + + @annotation.tailrec + private val transcode: ((Array[Char], Char, Array[Char], Array[Char]) => Array[Char]) = (l, c, r, o) => + if (c == '\0' && r.length == 0) o + else { + def shift(d: Int, ca: Array[Char]) = { + val sca = r.splitAt(d - 1) + + ( + if (sca._1.length > 0) (l :+ c) ++ sca._1 else l :+ c, + if (sca._2.length > 0) sca._2.head else '\0', + if (sca._2.length > 1) sca._2.tail else Array.empty[Char], + ca + ) + } + + val t = { + (c: @annotation.switch) match { + case 'a' | 'e' | 'i' | 'o' | 'u' => if (l.length == 0) shift(1, o:+ c) else shift(1, o) + case 'f' | 'j' | 'l' | 'm' | 'n' | 'r' => shift(1, o :+ c) + case 'b' => if (l.length >= 1 && l.last == 'm' && r.length == 0) shift(1, o) else shift(1, o :+ 'b') + case 'c' => + if (r.length >= 1 && r.head == 'h' && l.length >= 1 && l.last == 's') shift(1, o :+ 'k') + else if (r.length >= 2 && r.head == 'i' && r(1) == 'a') shift(3, o :+ 'x') + else if ((r.length >= 1 && r.head == 'h') + || (l.length >= 1 && r.length >= 1 && l.last == 's' && r.head == 'h')) shift(2, o :+ 'x') + else if (l.length >= 1 && r.length >= 1 && l.last == 's' + && (r.head == 'i' || r.head == 'e' || r.head == 'y')) shift(1, o) + else if (r.length >= 1 && (r.head == 'i' || r.head == 'e' || r.head == 'y')) shift(1, o :+ 's') + else shift(1, o :+ 'k') + case 'd' => + if (r.length >= 2 && r.head == 'g' + && (r(1) == 'e' || r(1) == 'y' || r(1) == 'i')) shift(1, o :+ 'j') + else shift(1, o :+ 't') + case 'g' => + if ((r.length > 1 && r.head == 'h') + || (r.length == 1 && r.head == 'n') + || (r.length == 3 && r.head == 'n' && r(1) == 'e' && r(2) == 'd')) shift(1, o) + else if (r.length >= 1 && (r.head == 'i' || r.head == 'e' || r.head == 'y')) shift(2, o :+ 'j') + else shift(1, o :+ 'k') + case 'h' => + if ((l.length >= 1 && (LowercaseVowel isSuperset l.last) && (r.length == 0 || !(LowercaseVowel isSuperset r.head))) + || (l.length >= 2 && l.last == 'h' + && (l(l.length - 2) == 'c' || l(l.length - 2) == 's' || l(l.length - 2) == 'p' + || l(l.length - 2) == 't' || l(l.length - 2) == 'g'))) shift(1, o) + else shift(1, o :+ 'h') + case 'k' => if (l.length >= 1 && l.last == 'c') shift(1, o) else shift(1, o :+ 'k') + case 'p' => if (r.length >= 1 && r.head == 'h') shift(2, o :+ 'f') else shift(1, o :+ 'p') + case 'q' => shift(1, o :+ 'k') + case 's' => + if (r.length >= 2 && r.head == 'i' && (r(1) == 'o' || r(1) == 'a')) shift(3, o :+ 'x') + else if (r.length >= 1 && r.head == 'h') shift(2, o :+ 'x') + else shift(1, o :+ 's') + case 't' => + if (r.length >= 2 && r.head == 'i' && (r(1) == 'a' || r(1) == 'o')) shift(3, o :+ 'x') + else if (r.length >= 1 && r.head == 'h') shift(2, o :+ '0') + else if (r.length >= 2 && r.head == 'c' && r(1) == 'h') shift(1, o) + else shift(1, o :+ 't') + case 'v' => shift(1, o :+ 'f') + case 'w' | 'y' => if (r.length == 0 || !(LowercaseVowel isSuperset r.head)) shift(1, o) else shift(1, o :+ c) + case 'x' => shift(1, (o :+ 'k') :+ 's') + case 'z' => shift(1, o :+ 's') + case _ => shift(1, o) + } + } + + transcode(t._1, t._2, t._3, t._4) + } + + private val transcodeHead: (Array[Char] => Array[Char]) = (ca) => + (ca.length: @annotation.switch) match { + case 0 => ca + case 1 => if (ca.head == 'x') Array('s') else ca + case _ => + (ca.head: @annotation.switch) match { + case 'a' if ca(1) == 'e' => ca.tail + case 'g' | 'k' | 'p' if ca(1) == 'n' => ca.tail + case 'w' if ca(1) == 'r' => ca.tail + case 'w' if ca(1) == 'h' => 'w' +: ca.drop(2) + case 'x' => 's' +: ca.tail + case _ => ca + } + } +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/phonetic/MetaphoneMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/MetaphoneMetric.scala new file mode 100755 index 0000000..d06f774 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/MetaphoneMetric.scala @@ -0,0 +1,15 @@ +package com.rockymadden.stringmetric.phonetic + +import com.rockymadden.stringmetric.Metric.StringMetric + +case object MetaphoneMetric extends StringMetric[Boolean] { + import com.rockymadden.stringmetric.Alphabet.Alpha + + override def compare(a: Array[Char], b: Array[Char]): Option[Boolean] = + if (a.length == 0 || !(Alpha isSuperset a.head) || b.length == 0 || !(Alpha isSuperset b.head)) None + else MetaphoneAlgorithm.compute(a).filter(_.length > 0).flatMap { mp1 => + MetaphoneAlgorithm.compute(b).filter(_.length > 0).map(mp1.sameElements(_)) + } + + override def compare(a: String, b: String): Option[Boolean] = compare(a.toCharArray, b.toCharArray) +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/phonetic/NysiisAlgorithm.scala b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/NysiisAlgorithm.scala new file mode 100755 index 0000000..3e46675 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/NysiisAlgorithm.scala @@ -0,0 +1,115 @@ +package com.rockymadden.stringmetric.phonetic + +import com.rockymadden.stringmetric.Algorithm.StringAlgorithm + +case object NysiisAlgorithm extends StringAlgorithm { + import com.rockymadden.stringmetric.Alphabet.{Alpha, LowercaseVowel} + + override def compute(a: Array[Char]): Option[Array[Char]] = + if (a.length == 0 || !(Alpha isSuperset a.head)) None + else { + val tr = transcodeRight(a.map(_.toLower)) + val tl = transcodeLeft(tr._1) + val t = + if (tl._2.length == 0) tl._1 ++ tr._2 + else tl._1 ++ transcodeCenter( + Array.empty[Char], + tl._2.head, + if (tl._2.length > 1) tl._2.tail else Array.empty[Char], + Array.empty[Char] + ) ++ tr._2 + + if (t.length == 1) Some(t) + else Some(t.head +: (cleanLast andThen cleanTerminal andThen deduplicate)(t.tail)) + } + + override def compute(a: String): Option[String] = compute(a.toCharArray).map(_.mkString) + + private val cleanLast: (Array[Char] => Array[Char]) = (ca) => + if (ca.length == 0) ca + else if(ca.last == 'a' || ca.last == 's') + ca.dropRight(ca.reverseIterator.takeWhile(c => c == 'a' || c == 's').length) + else ca + + private val cleanTerminal: (Array[Char] => Array[Char]) = (ca) => + if (ca.length >= 2 && ca.last == 'y' && ca(ca.length - 2) == 'a') ca.dropRight(2) :+ 'y' + else ca + + private val deduplicate: (Array[Char] => Array[Char]) = (ca) => + if (ca.length <= 1) ca + else ca.sliding(2).withFilter(a => a(0) != a(1)).map(_(0)).toArray[Char] :+ ca.last + + @annotation.tailrec + private val transcodeCenter: ((Array[Char], Char, Array[Char], Array[Char]) => Array[Char]) = (l, c, r, o) => + if (c == '\0' && r.length == 0) o + else { + def shift(d: Int, ca: Array[Char]) = { + val sca = r.splitAt(d - 1) + + ( + if (sca._1.length > 0) (l :+ c) ++ sca._1 else l :+ c, + if (sca._2.length > 0) sca._2.head else '\0', + if (sca._2.length > 1) sca._2.tail else Array.empty[Char], + ca + ) + } + + val t = { + (c: @annotation.switch) match { + case 'a' | 'i' | 'o' | 'u' => shift(1, o :+ 'a') + case 'b' | 'c' | 'd' | 'f' | 'g' | 'j' | 'l' | 'n' | 'r' | 't' | 'v' | 'x' | 'y' => shift(1, o :+ c) + case 'e' => + if (r.length >= 1 && r.head == 'v') shift(2, o ++ Array('a', 'f')) + else shift(1, o :+ 'a') + case 'h' => + if (l.length >= 1 && (!(LowercaseVowel isSuperset l.last) || (r.length >= 1 && !(LowercaseVowel isSuperset r.head)))) + shift(1, o) + else shift(1, o :+ c) + case 'k' => if (r.length >= 1 && r.head == 'n') shift(2, o :+ 'n') else shift(1, o :+ 'c') + case 'm' => shift(1, o :+ 'n') + case 'p' => if (r.length >= 1 && r.head == 'h') shift(2, o :+ 'f') else shift(1, o :+ c) + case 'q' => shift(1, o :+ 'g') + case 's' => + if (r.length >= 2 && r.head == 'c' && r(1) == 'h') shift(3, o :+ c) + else shift(1, o :+ c) + case 'w' => + if (l.length >= 1 && (LowercaseVowel isSuperset l.last)) shift(1, o) + else shift(1, o :+ c) + case 'z' => shift(1, o :+ 's') + case _ => shift(1, o) + } + } + + transcodeCenter(t._1, t._2, t._3, t._4) + } + + private val transcodeLeft: (Array[Char] => (Array[Char], Array[Char])) = (ca) => + if (ca.length == 0) (Array.empty[Char], ca) + else { + lazy val tr2 = ca.takeRight(ca.length - 2) + lazy val tr3 = ca.takeRight(ca.length - 3) + + (ca.head: @annotation.switch) match { + case 'k' if ca.length >= 2 && ca(1) == 'n' => (Array('n', 'n'), tr2) + case 'k' => (Array('c'), ca.tail) + case 'm' if ca.length >= 3 && (ca(1) == 'a' && ca(2) == 'c') => (Array('m', 'c'), tr3) + case 'p' if ca.length >= 2 && (ca(1) == 'h' || ca(1) == 'f') => (Array('f', 'f'), tr2) + case 's' if ca.length >= 3 && (ca(1) == 'c' && ca(2) == 'h') => (Array('s', 's'), tr3) + case _ => (Array(ca.head), ca.tail) + } + } + + private val transcodeRight: (Array[Char] => (Array[Char], Array[Char])) = (ca) => + if (ca.length >= 2) { + val lc = ca(ca.length - 1) + val lcm1 = ca(ca.length - 2) + lazy val t2 = ca.take(ca.length - 2) + + (lc: @annotation.switch) match { + case 'd' if lcm1 == 'n' || lcm1 == 'r' => (t2, Array('d')) + case 'e' if lcm1 == 'e' || lcm1 == 'i' => (t2, Array('y')) + case 't' if lcm1 == 'd' || lcm1 == 'n' || lcm1 == 'r' => (t2, Array('d')) + case _ => (ca, Array.empty[Char]) + } + } else (ca, Array.empty[Char]) +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/phonetic/NysiisMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/NysiisMetric.scala new file mode 100755 index 0000000..c9a0914 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/NysiisMetric.scala @@ -0,0 +1,24 @@ +package com.rockymadden.stringmetric.phonetic + +import com.rockymadden.stringmetric.Metric.StringMetric + +case object NysiisMetric extends StringMetric[Boolean] { + import com.rockymadden.stringmetric.Alphabet.Alpha + + override def compare(a: Array[Char], b: Array[Char]): Option[Boolean] = { + val unequal: ((Char, Char) => Boolean) = (c1, c2) => { + val lc1 = c1.toLower + val lc2 = c2.toLower + + (if (lc1 == 'k') 'c' else lc1) != (if (lc2 == 'k') 'c' else lc2) + } + + if (a.length == 0 || !(Alpha isSuperset a.head) || b.length == 0 || !(Alpha isSuperset b.head)) None + else if (unequal(a.head, b.head)) Some(false) + else NysiisAlgorithm.compute(a).filter(_.length > 0).flatMap { ny1 => + NysiisAlgorithm.compute(b).filter(_.length > 0).map(ny1.sameElements(_)) + } + } + + override def compare(a: String, b: String): Option[Boolean] = compare(a.toCharArray, b.toCharArray) +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/phonetic/RefinedNysiisAlgorithm.scala b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/RefinedNysiisAlgorithm.scala new file mode 100755 index 0000000..9976847 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/RefinedNysiisAlgorithm.scala @@ -0,0 +1,121 @@ +package com.rockymadden.stringmetric.phonetic + +import com.rockymadden.stringmetric.Algorithm.StringAlgorithm + +case object RefinedNysiisAlgorithm extends StringAlgorithm { + import com.rockymadden.stringmetric.Alphabet.{Alpha, LowercaseVowel} + + override def compute(a: Array[Char]): Option[Array[Char]] = + if (a.length == 0 || !(Alpha isSuperset a.head)) None + else { + val lca = a.map(_.toLower) + val tlh = (transcodeHead andThen transcodeLast)(lca.head +: cleanLast(lca.tail, Set('s', 'z'))) + val t = transcode(Array.empty[Char], tlh.head, tlh.tail, Array.empty[Char]) + + if (t.length == 1) Some(t) + else Some(deduplicate( + t.head +: (cleanLast.tupled andThen cleanTerminal)(t.tail, Set('a')) + )) + } + + override def compute(string: String): Option[String] = compute(string.toCharArray).map(_.mkString) + + private val cleanLast: ((Array[Char], Set[Char]) => Array[Char]) = (ca, s) => + if (ca.length == 0) ca + else if(s.contains(ca.last)) ca.dropRight(ca.reverseIterator.takeWhile(c => s.contains(c)).length) + else ca + + private val cleanTerminal: (Array[Char] => Array[Char]) = (ca) => + if (ca.length >= 2 && ca.last == 'y' && ca(ca.length - 2) == 'a') ca.dropRight(2) :+ 'y' + else ca + + private val deduplicate: (Array[Char] => Array[Char]) = (ca) => + if (ca.length <= 1) ca + else ca.sliding(2).withFilter(a => a(0) != a(1)).map(a => a(0)).toArray[Char] :+ ca.last + + @annotation.tailrec + private val transcode: ((Array[Char], Char, Array[Char], Array[Char]) => Array[Char]) = (l, c, r, o) => + if (c == '\0' && r.length == 0) o + else { + def shift(d: Int, ca: Array[Char]) = { + val sca = r.splitAt(d - 1) + + ( + if (sca._1.length > 0) (l :+ c) ++ sca._1 else l :+ c, + if (sca._2.length > 0) sca._2.head else '\0', + if (sca._2.length > 1) sca._2.tail else Array.empty[Char], + ca + ) + } + + val t = { + (c: @annotation.switch) match { + case 'a' | 'i' | 'o' | 'u' => + if (l.length == 0) shift(1, o :+ c) + else shift(1, o :+ 'a') + case 'b' | 'c' | 'f' | 'j' | 'l' | 'n' | 'r' | 't' | 'v' | 'x' => shift(1, o :+ c) + case 'd' => + if (r.length >= 1 && r.head == 'g') shift(2, o :+ 'g') else shift(1, o :+ c) + case 'e' => + if (l.length == 0) shift(1, o :+ c) + else if (r.length >= 1 && r.head == 'v') shift(2, o ++ Array('a', 'f')) + else shift(1, o :+ 'a') + case 'g' => + if (r.length >= 2 && r.head == 'h' && r(1) == 't') shift(3, o ++ Array('g', 't')) + else shift(1, o :+ c) + case 'h' => + if (l.length == 0) shift(1, o :+ c) + else if (!(LowercaseVowel isSuperset l.last) || (r.length >= 1 && !(LowercaseVowel isSuperset r.head))) + shift(1, o) + else shift(1, o :+ c) + case 'k' => if (r.length >= 1 && r.head == 'n') shift(2, o :+ 'n') else shift(1, o :+ 'c') + case 'm' => if (l.length == 0) shift(1, o :+ c) else shift(1, o :+ 'n') + case 'p' => if (r.length >= 1 && r.head == 'h') shift(2, o :+ 'f') else shift(1, o :+ c) + case 'q' => if (l.length == 0) shift(1, o :+ c) else shift(1, o :+ 'g') + case 's' => + if (r.length >= 2 && r.head == 'c' && r(1) == 'h') shift(3, o :+ c) + else if (r.length >= 1 && r.head == 'h') shift(2, o :+ c) + else shift(1, o :+ c) + case 'w' => + if (l.length >= 1 && (LowercaseVowel isSuperset l.last)) shift(1, o) + else if (r.length >= 1 && r.head == 'r') shift(2, o :+ 'r') + else shift(1, o :+ c) + case 'y' => + if (l.length >= 1 && r.length >= 2 && r.head == 'w') shift(2, o :+ 'a') + else if (r.length >= 1 && r.head == 'w') shift(2, o :+ c) + else if (l.length >= 1 && r.length >= 1) shift(1, o :+ 'a') + else shift(1, o :+ c) + case 'z' => if (l.length == 0) shift(1, o :+ c) else shift(1, o :+ 's') + case _ => shift(1, o) + } + } + + transcode(t._1, t._2, t._3, t._4) + } + + private val transcodeHead: (Array[Char] => Array[Char]) = (ca) => + if (ca.length == 0) ca + else + (ca.head: @annotation.switch) match { + case 'm' if ca.length >= 3 && ca(1) == 'a' && ca(2) == 'c' => + Array('m', 'c') ++ ca.takeRight(ca.length - 3) + case 'p' if ca.length >= 2 && ca(1) == 'f' =>'f' +: ca.takeRight(ca.length - 2) + case _ => ca + } + + private val transcodeLast: (Array[Char] => Array[Char]) = (ca) => + if (ca.length >= 2) { + val lc = ca(ca.length - 1) + val lcm1 = ca(ca.length - 2) + lazy val t2 = ca.take(ca.length - 2) + + (lc: @annotation.switch) match { + case 'd' if lcm1 == 'n' || lcm1 == 'r' => t2 :+ 'd' + case 'e' if lcm1 == 'e' || lcm1 == 'i' || lcm1 =='y' => t2 :+ 'y' + case 't' if lcm1 == 'd' || lcm1 == 'n' || lcm1 == 'r' => t2 :+ 'd' + case 'x' if lcm1 == 'e' => t2 ++ Array('e', 'c') + case 'x' if lcm1 == 'i' => t2 ++ Array('i', 'c') + case _ => ca + } + } else ca +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/phonetic/RefinedNysiisMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/RefinedNysiisMetric.scala new file mode 100755 index 0000000..488f261 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/RefinedNysiisMetric.scala @@ -0,0 +1,24 @@ +package com.rockymadden.stringmetric.phonetic + +import com.rockymadden.stringmetric.Metric.StringMetric + +case object RefinedNysiisMetric extends StringMetric[Boolean] { + import com.rockymadden.stringmetric.Alphabet.Alpha + + override def compare(a: Array[Char], b: Array[Char]): Option[Boolean] = { + val unequal = (c1: Char, c2: Char) => { + val lc1 = c1.toLower + val lc2 = c2.toLower + + (if (lc1 == 'k') 'c' else lc1) != (if (lc2 == 'k') 'c' else lc2) + } + + if (a.length == 0 || !(Alpha isSuperset a.head) || b.length == 0 || !(Alpha isSuperset b.head)) None + else if (unequal(a.head, b.head)) Some(false) + else RefinedNysiisAlgorithm.compute(a).filter(_.length > 0).flatMap { rny1 => + RefinedNysiisAlgorithm.compute(b).filter(_.length > 0).map(rny1.sameElements(_)) + } + } + + override def compare(a: String, b: String): Option[Boolean] = compare(a.toCharArray, b.toCharArray) +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/phonetic/RefinedSoundexAlgorithm.scala b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/RefinedSoundexAlgorithm.scala new file mode 100755 index 0000000..e8f3af6 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/RefinedSoundexAlgorithm.scala @@ -0,0 +1,59 @@ +package com.rockymadden.stringmetric.phonetic + +import com.rockymadden.stringmetric.Algorithm.StringAlgorithm + +case object RefinedSoundexAlgorithm extends StringAlgorithm { + import com.rockymadden.stringmetric.Alphabet.Alpha + + override def compute(a: Array[Char]): Option[Array[Char]] = + if (a.length == 0 || !(Alpha isSuperset a.head)) None + else Some(transcode(a, Array(a.head.toLower))) + + override def compute(a: String): Option[String] = compute(a.toCharArray).map(_.mkString) + + @annotation.tailrec + private val transcode: ((Array[Char], Array[Char]) => Array[Char]) = (i, o) => + if (i.length == 0) o + else { + val c = i.head.toLower + val m2 = (mc: Char) => (mc: @annotation.switch) match { + case 'a' | 'e' | 'h' | 'i' | 'o' | 'u' | 'w' | 'y' => '0' + case 'b' | 'p' => '1' + case 'f' | 'v' => '2' + case 'c' | 'k' | 's' => '3' + case 'g' | 'j' => '4' + case 'q' | 'x' | 'z' => '5' + case 'd' | 't' => '6' + case 'l' => '7' + case 'm' | 'n' => '8' + case 'r' => '9' + case _ => '\0' + } + val m1 = (mc: Char, pc: Char) => (mc: @annotation.switch) match { + case 'a' | 'e' | 'h' | 'i' | 'o' | 'u' | 'w' | 'y' if pc != '0' => '0' + case 'b' | 'p' if pc != '1' => '1' + case 'f' | 'v' if pc != '2' => '2' + case 'c' | 'k' | 's' if pc != '3' => '3' + case 'g' | 'j' if pc != '4' => '4' + case 'q' | 'x' | 'z' if pc != '5' => '5' + case 'd' | 't' if pc != '6' => '6' + case 'l' if pc != '7' => '7' + case 'm' | 'n' if pc != '8' => '8' + case 'r' if pc != '9' => '9' + case _ => '\0' + } + val a = + // Code twice. + if (o.length == 1) m2(c) + // Code once. + else m1( + c, + (o.last: @annotation.switch) match { + case '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' => o.last + case _ => m2(o.last) + } + ) + + transcode(i.tail, if (a != '\0') o :+ a else o) + } +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/phonetic/RefinedSoundexMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/RefinedSoundexMetric.scala new file mode 100755 index 0000000..289fe29 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/RefinedSoundexMetric.scala @@ -0,0 +1,16 @@ +package com.rockymadden.stringmetric.phonetic + +import com.rockymadden.stringmetric.Metric.StringMetric + +case object RefinedSoundexMetric extends StringMetric[Boolean] { + import com.rockymadden.stringmetric.Alphabet.Alpha + + override def compare(a: Array[Char], b: Array[Char]): Option[Boolean] = + if (a.length == 0 || !(Alpha isSuperset a.head) || b.length == 0 || !(Alpha isSuperset b.head)) None + else if (a.head.toLower != b.head.toLower) Some(false) + else RefinedSoundexAlgorithm.compute(a).filter(_.length > 0).flatMap { rse1 => + RefinedSoundexAlgorithm.compute(b).filter(_.length > 0).map(rse1.sameElements(_)) + } + + override def compare(a: String, b: String): Option[Boolean] = compare(a.toCharArray, b.toCharArray) +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/phonetic/SoundexAlgorithm.scala b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/SoundexAlgorithm.scala new file mode 100755 index 0000000..b211908 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/SoundexAlgorithm.scala @@ -0,0 +1,57 @@ +package com.rockymadden.stringmetric.phonetic + +import com.rockymadden.stringmetric.Algorithm.StringAlgorithm + +case object SoundexAlgorithm extends StringAlgorithm { + import com.rockymadden.stringmetric.Alphabet.Alpha + + override def compute(a: Array[Char]): Option[Array[Char]] = + if (a.length == 0 || !(Alpha isSuperset a.head)) None + else { + val fc = a.head.toLower + + Some(transcode(a.tail, fc, Array(fc)).padTo(4, '0')) + } + + override def compute(string: String): Option[String] = compute(string.toCharArray).map(_.mkString) + + @annotation.tailrec + private val transcode: ((Array[Char], Char, Array[Char]) => Array[Char]) = (i, pc, o) => + if (i.length == 0) o + else { + val c = i.head.toLower + val m2 = (mc: Char) => (mc: @annotation.switch) match { + case 'b' | 'f' | 'p' | 'v' => '1' + case 'c' | 'g' | 'j' | 'k' | 'q' | 's' | 'x' | 'z' => '2' + case 'd' | 't' => '3' + case 'l' => '4' + case 'm' | 'n' => '5' + case 'r' => '6' + case _ => '\0' + } + val m1 = (mc: Char, pc: Char) => (mc: @annotation.switch) match { + case 'b' | 'f' | 'p' | 'v' if pc != '1' => '1' + case 'c' | 'g' | 'j' | 'k' | 'q' | 's' | 'x' | 'z' if pc != '2' => '2' + case 'd' | 't' if pc != '3' => '3' + case 'l' if pc != '4' => '4' + case 'm' | 'n' if pc != '5' => '5' + case 'r' if pc != '6' => '6' + case _ => '\0' + } + val a = pc match { + // Code twice. + case 'a' | 'e' | 'i' | 'o' | 'u' | 'y' => m2(c) + // Code once. + case _ => m1( + c, + (o.last: @annotation.switch) match { + case '1' | '2' | '3' | '4' | '5' | '6' => o.last + case _ => m2(o.last) + } + ) + } + + if (o.length == 3 && a != '\0') o :+ a + else transcode(i.tail, c, if (a != '\0') o :+ a else o) + } +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/phonetic/SoundexMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/SoundexMetric.scala new file mode 100755 index 0000000..eca32db --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/phonetic/SoundexMetric.scala @@ -0,0 +1,16 @@ +package com.rockymadden.stringmetric.phonetic + +import com.rockymadden.stringmetric.Metric.StringMetric + +case object SoundexMetric extends StringMetric[Boolean] { + import com.rockymadden.stringmetric.Alphabet.Alpha + + override def compare(a: Array[Char], b: Array[Char]): Option[Boolean] = + if (a.length == 0 || !(Alpha isSuperset a.head) || b.length == 0 || !(Alpha isSuperset b.head)) None + else if (a.head.toLower != b.head.toLower) Some(false) + else SoundexAlgorithm.compute(a).filter(_.length > 0).flatMap { se1 => + SoundexAlgorithm.compute(b).filter(_.length > 0).map(se1.sameElements(_)) + } + + final override def compare(a: String, b: String): Option[Boolean] = compare(a.toCharArray, b.toCharArray) +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/similarity/DiceSorensenMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/similarity/DiceSorensenMetric.scala new file mode 100755 index 0000000..0ad3915 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/similarity/DiceSorensenMetric.scala @@ -0,0 +1,27 @@ +package com.rockymadden.stringmetric.similarity + +import com.rockymadden.stringmetric.Metric.StringMetric + +/** + * An implementation of the Dice/Sorensen metric. This implementation differs in that n-gram size is required. + * Traditionally, the algorithm uses bigrams. + */ +final case class DiceSorensenMetric(n: Int) extends StringMetric[Double] { + import com.rockymadden.stringmetric.Tokenize.NGramTokenizer + import com.rockymadden.stringmetric.MatchTuple + + override def compare(a: Array[Char], b: Array[Char]): Option[Double] = + if (n <= 0 || a.length < n || b.length < n) None // Because length is less than n, it is not possible to compare. + else if (a.sameElements(b)) Some(1d) + else NGramTokenizer(n).tokenize(a).flatMap { ca1bg => + NGramTokenizer(n).tokenize(b).map { ca2bg => + val ms = scoreMatches(ca1bg.map(_.mkString), ca2bg.map(_.mkString)) + + (2d * ms) / (ca1bg.length + ca2bg.length) + } + } + + override def compare(a: String, b: String): Option[Double] = compare(a.toCharArray, b.toCharArray) + + private val scoreMatches: (MatchTuple[String] => Int) = (mt) => mt._1.intersect(mt._2).length +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/similarity/HammingMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/similarity/HammingMetric.scala new file mode 100755 index 0000000..4a90f32 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/similarity/HammingMetric.scala @@ -0,0 +1,18 @@ +package com.rockymadden.stringmetric.similarity + +import com.rockymadden.stringmetric.Metric.StringMetric + +case object HammingMetric extends StringMetric[Int] { + import com.rockymadden.stringmetric.CompareTuple + + override def compare(a: Array[Char], b: Array[Char]): Option[Int] = + if (a.length == 0 || b.length == 0 || a.length != b.length) None + else if (a.sameElements(b)) Some(0) + else Some(hamming(a, b)) + + override def compare(a: String, b: String): Option[Int] = compare(a.toCharArray, b.toCharArray) + + private val hamming: (CompareTuple[Char] => Int) = (ct) => + if (ct._1.length == 0) 0 + else ct._1.zip(ct._2).count(t => t._1 != t._2) +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/similarity/JaccardMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/similarity/JaccardMetric.scala new file mode 100755 index 0000000..6ec5db4 --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/similarity/JaccardMetric.scala @@ -0,0 +1,20 @@ +package com.rockymadden.stringmetric.similarity + +import com.rockymadden.stringmetric.Metric.StringMetric + +final case class JaccardMetric(n: Int) extends StringMetric[Double] { + import com.rockymadden.stringmetric.Tokenize.NGramTokenizer + + override def compare(a: Array[Char], b: Array[Char]): Option[Double] = + if (n <= 0 || a.length < n || b.length < n) None // Because length is less than n, it is not possible to compare. + else if (a.sameElements(b)) Some(1d) + else NGramTokenizer(n).tokenize(a).flatMap { ca1bg => + NGramTokenizer(n).tokenize(b).map { ca2bg => + val i = (ca1bg.map(_.mkString) intersect ca2bg.map(_.mkString)).length + + i.toDouble / (ca1bg.length + ca2bg.length - i) + } + } + + override def compare(a: String, b: String): Option[Double] = compare(a.toCharArray, b.toCharArray) +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/similarity/JaroMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/similarity/JaroMetric.scala new file mode 100755 index 0000000..575d67a --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/similarity/JaroMetric.scala @@ -0,0 +1,66 @@ +package com.rockymadden.stringmetric.similarity + +import com.rockymadden.stringmetric.Metric.StringMetric +import scala.Some + +/** + * An implementation of the Jaro metric. One differing detail in this implementation is that if a character is matched + * in string2, it cannot be matched upon again. This results in a more penalized distance in these scenarios. + */ +case object JaroMetric extends StringMetric[Double] { + import com.rockymadden.stringmetric.{CompareTuple, MatchTuple} + import scala.collection.mutable.{ArrayBuffer, HashSet} + + override def compare(a: Array[Char], b: Array[Char]): Option[Double] = + if (a.length == 0 || b.length == 0) None + else if (a.sameElements(b)) Some(1d) + else { + val mt = `match`(a, b) + val ms = scoreMatches(mt._1, mt._2) + + if (ms == 0) Some(0d) + else { + val ts = scoreTranspositions(mt._1, mt._2) + + Some(((ms.toDouble / a.length) + (ms.toDouble / b.length) + ((ms.toDouble - ts) / ms)) / 3) + } + } + + override def compare(a: String, b: String): Option[Double] = compare(a.toCharArray, b.toCharArray) + + private val `match`: (CompareTuple[Char] => MatchTuple[Char]) = (ct) => { + lazy val window = math.abs((math.max(ct._1.length, ct._2.length) / 2d).floor.toInt - 1) + val one = ArrayBuffer.empty[Int] + val two = HashSet.empty[Int] + var i = 0 + var bi = false + + while (i < ct._1.length && !bi) { + val start = if (i - window <= 0) 0 else i - window + val end = if (i + window >= ct._2.length - 1) ct._2.length - 1 else i + window + + if (start > ct._2.length - 1) bi = !bi + else { + var ii = start + var bii = false + + while (ii <= end && !bii) { + if (!two.contains(ii) && ct._1(i) == ct._2(ii)) { + one += i + two += ii + bii = !bii + } else ii += 1 + } + + i += 1 + } + } + + (one.toArray.map(ct._1(_)), two.toArray.sortWith(_ < _).map(ct._2(_))) + } + + private val scoreMatches: (MatchTuple[Char] => Int) = (mt) => mt._1.length + + private val scoreTranspositions: (MatchTuple[Char] => Int) = (mt) => + (mt._1.zip(mt._2).count(t => t._1 != t._2) / 2d).floor.toInt +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/similarity/JaroWinklerMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/similarity/JaroWinklerMetric.scala new file mode 100755 index 0000000..e83f73f --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/similarity/JaroWinklerMetric.scala @@ -0,0 +1,23 @@ +package com.rockymadden.stringmetric.similarity + +import com.rockymadden.stringmetric.Metric.StringMetric + +/** + * An implementation of the Jaro-Winkler metric. One differing detail in this implementation is that if a character is + * matched in string2, it cannot be matched upon again. This results in a more penalized distance in these scenarios + * (e.g. comparing henka and henkan distance is 0.9666 versus the typical 0.9722). + */ +case object JaroWinklerMetric extends StringMetric[Double] { + override def compare(a: Array[Char], b: Array[Char]): Option[Double] = + JaroMetric.compare(a, b).map { + case 0d => 0d + case 1d => 1d + case jaro => { + val prefix = a.zip(b).takeWhile(t => t._1 == t._2) + + jaro + ((if (prefix.length <= 4) prefix.length else 4) * 0.1d * (1 - jaro)) + } + } + + override def compare(a: String, b: String): Option[Double] = compare(a.toCharArray, b.toCharArray) +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/similarity/LevenshteinMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/similarity/LevenshteinMetric.scala new file mode 100755 index 0000000..fb90cdc --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/similarity/LevenshteinMetric.scala @@ -0,0 +1,40 @@ +package com.rockymadden.stringmetric.similarity + +import com.rockymadden.stringmetric.Metric.StringMetric + +case object LevenshteinMetric extends StringMetric[Int] { + import com.rockymadden.stringmetric.CompareTuple + + override def compare(a: Array[Char], b: Array[Char]): Option[Int] = + if (a.length == 0 || b.length == 0) None + else if (a.sameElements(b)) Some(0) + else Some(levenshtein(a, b)) + + override def compare(a: String, b: String): Option[Int] = compare(a.toCharArray, b.toCharArray) + + private val levenshtein: (CompareTuple[Char] => Int) = (ct) => { + val m = Array.fill[Int](ct._1.length + 1, ct._2.length + 1)(-1) + + def distance(t: (Int, Int)): Int = t match { + case (r, 0) => r + case (0, c) => c + case (r, c) if m(r)(c) != -1 => m(r)(c) + case (r, c) => { + val min = + if (ct._1(r - 1) == ct._2(c - 1)) distance(r - 1, c - 1) + else math.min( + math.min( + distance(r - 1, c) + 1, // Delete (left). + distance(r, c - 1) + 1 // Insert (up). + ), + distance(r - 1, c - 1) + 1 // Substitute (left-up). + ) + + m(r)(c) = min + min + } + } + + distance(ct._1.length, ct._2.length) + } +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/similarity/NGramMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/similarity/NGramMetric.scala new file mode 100755 index 0000000..8c194ce --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/similarity/NGramMetric.scala @@ -0,0 +1,24 @@ +package com.rockymadden.stringmetric.similarity + +import com.rockymadden.stringmetric.Metric.StringMetric + +final case class NGramMetric(n: Int) extends StringMetric[Double] { + import com.rockymadden.stringmetric.MatchTuple + import com.rockymadden.stringmetric.Tokenize.NGramTokenizer + import scala.math + + override def compare(a: Array[Char], b: Array[Char]): Option[Double] = + if (n <= 0 || a.length < n || b.length < n) None // Because length is less than n, it is not possible to compare. + else if (a.sameElements(b)) Some(1d) + else NGramTokenizer(n).tokenize(a).flatMap { ca1bg => + NGramTokenizer(n).tokenize(b).map { ca2bg => + val ms = scoreMatches((ca1bg.map(_.mkString), ca2bg.map(_.mkString))) + + ms.toDouble / math.max(ca1bg.length, ca2bg.length) + } + } + + override def compare(a: String, b: String): Option[Double] = compare(a.toCharArray, b.toCharArray) + + private val scoreMatches: (MatchTuple[String] => Int) = (mt) => mt._1.intersect(mt._2).length +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/similarity/OverlapMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/similarity/OverlapMetric.scala new file mode 100755 index 0000000..8f0418b --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/similarity/OverlapMetric.scala @@ -0,0 +1,24 @@ +package com.rockymadden.stringmetric.similarity + +import com.rockymadden.stringmetric.Metric.StringMetric + +final case class OverlapMetric(n: Int) extends StringMetric[Double] { + import com.rockymadden.stringmetric.MatchTuple + import com.rockymadden.stringmetric.Tokenize.NGramTokenizer + import scala.math + + override def compare(a: Array[Char], b: Array[Char]): Option[Double] = + if (n <= 0 || a.length < n || b.length < n) None // Because length is less than n, it is not possible to compare. + else if (a.sameElements(b)) Some(1d) + else NGramTokenizer(n).tokenize(a).flatMap { ca1bg => + NGramTokenizer(n).tokenize(b).map { ca2bg => + val ms = scoreMatches(ca1bg.map(_.mkString), ca2bg.map(_.mkString)) + + ms.toDouble / math.min(ca1bg.length, ca2bg.length) + } + } + + override def compare(a: String, b: String): Option[Double] = compare(a.toCharArray, b.toCharArray) + + private val scoreMatches: (MatchTuple[String] => Int) = (mt) => mt._1.intersect(mt._2).length +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/similarity/RatcliffObershelpMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/similarity/RatcliffObershelpMetric.scala new file mode 100755 index 0000000..fa113bc --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/similarity/RatcliffObershelpMetric.scala @@ -0,0 +1,43 @@ +package com.rockymadden.stringmetric.similarity + +import com.rockymadden.stringmetric.Metric.StringMetric + +case object RatcliffObershelpMetric extends StringMetric[Double] { + import com.rockymadden.stringmetric.CompareTuple + + override def compare(a: Array[Char], b: Array[Char]): Option[Double] = + if (a.length == 0 || b.length == 0) None + else if (a.sameElements(b)) Some(1d) + else Some(2d * commonSequences(a, b).foldLeft(0)(_ + _.length) / (a.length + b.length)) + + override def compare(a: String, b: String): Option[Double] = compare(a.toCharArray, b.toCharArray) + + private def longestCommonSubsequence(ct: CompareTuple[Char]) = { + val m = Array.ofDim[Int](ct._1.length + 1, ct._2.length + 1) + var lrc = (0, 0, 0) // Length, row, column. + + for (r <- 0 to ct._1.length - 1; c <- 0 to ct._2.length - 1) { + if (ct._1(r) == ct._2(c)) { + val l = m(r)(c) + 1 + m(r + 1)(c + 1) = l + if (l > lrc._1) lrc = (l, r + 1, c + 1) + } + } + + lrc + } + + private val commonSequences: (CompareTuple[Char] => Array[Array[Char]]) = (ct) => { + val lcs = longestCommonSubsequence(ct) + + if (lcs._1 == 0) Array.empty + else { + val sct1 = (ct._1.take(lcs._2 - lcs._1), ct._1.takeRight(ct._1.length - lcs._2)) + val sct2 = (ct._2.take(lcs._3 - lcs._1), ct._2.takeRight(ct._2.length - lcs._3)) + + Array(ct._1.slice(lcs._2 - lcs._1, lcs._2)) ++ + commonSequences(sct1._1, sct2._1) ++ + commonSequences(sct1._2, sct2._2) + } + } +} diff --git a/core/src/main/scala/com/rockymadden/stringmetric/similarity/WeightedLevenshteinMetric.scala b/core/src/main/scala/com/rockymadden/stringmetric/similarity/WeightedLevenshteinMetric.scala new file mode 100755 index 0000000..ae6f49c --- /dev/null +++ b/core/src/main/scala/com/rockymadden/stringmetric/similarity/WeightedLevenshteinMetric.scala @@ -0,0 +1,36 @@ +package com.rockymadden.stringmetric.similarity + +import com.rockymadden.stringmetric.Metric.StringMetric + +final case class WeightedLevenshteinMetric(delete: BigDecimal, insert: BigDecimal, substitute: BigDecimal) + extends StringMetric[Double] { + + import com.rockymadden.stringmetric.CompareTuple + + override def compare(a: Array[Char], b: Array[Char]): Option[Double] = + if (a.length == 0 || b.length == 0) None + else if (a.sameElements(b)) Some(0d) + else Some(weightedLevenshtein((a, b), (delete, insert, substitute)).toDouble) + + override def compare(a: String, b: String): Option[Double] = compare(a.toCharArray, b.toCharArray) + + private val weightedLevenshtein: ((CompareTuple[Char], (BigDecimal, BigDecimal, BigDecimal)) => BigDecimal) = + (ct, w) => { + val m = Array.ofDim[BigDecimal](ct._1.length + 1, ct._2.length + 1) + + for (r <- 0 to ct._1.length) m(r)(0) = w._1 * r + for (c <- 0 to ct._2.length) m(0)(c) = w._2 * c + + for (r <- 1 to ct._1.length; c <- 1 to ct._2.length) { + m(r)(c) = + if (ct._1(r - 1) == ct._2(c - 1)) m(r - 1)(c - 1) + else (m(r - 1)(c) + w._1).min( // Delete (left). + (m(r)(c - 1) + w._2).min( // Insert (up). + m(r - 1)(c - 1) + w._3 // Substitute (left-up). + ) + ) + } + + m(ct._1.length)(ct._2.length) + } +} |