From 04a9cac84f0bb6d5f9d573539fbb54c81d689148 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 29 Oct 2017 12:23:14 -0700 Subject: Finished migrating over to using `PathRef` to propagate changes to file paths and folders. `sideHash` remains, but is only used for direct "do I re-evaluate" checks, and relies on the return values to force downstream targets to re-evaluate if necessary --- src/main/scala/forge/Evaluator.scala | 52 ++++++++++++------------------ src/main/scala/forge/Target.scala | 16 ++++++--- src/main/scala/forge/Util.scala | 27 +++++++++++++--- src/main/scala/forge/package.scala | 2 +- src/test/scala/forge/EvaluationTests.scala | 2 ++ src/test/scala/forge/Framework.scala | 4 +++ src/test/scala/forge/Main.scala | 20 ++++++------ 7 files changed, 71 insertions(+), 52 deletions(-) (limited to 'src') diff --git a/src/main/scala/forge/Evaluator.scala b/src/main/scala/forge/Evaluator.scala index 85bd5ac6..888f6639 100644 --- a/src/main/scala/forge/Evaluator.scala +++ b/src/main/scala/forge/Evaluator.scala @@ -1,7 +1,7 @@ package forge -import play.api.libs.json.{JsValue, Json} +import play.api.libs.json.{JsValue, Json, Reads} import scala.collection.mutable import ammonite.ops._ @@ -17,19 +17,17 @@ class Evaluator(workspacePath: Path, ) val evaluated = new MutableOSet[Target[_]] - val results = mutable.Map.empty[Target[_], Any] - val groupHashes = mutable.Map.empty[Int, Int] + val results = mutable.LinkedHashMap.empty[Target[_], Any] + for (groupIndex <- sortedGroups.keys()){ val group = sortedGroups.lookupKey(groupIndex) - val (inputsHash, newResults, newEvaluated) = evaluateGroupCached( + val (newResults, newEvaluated) = evaluateGroupCached( group, results, - groupHashes, sortedGroups ) evaluated.appendAll(newEvaluated) for((k, v) <- newResults) results.put(k, v) - groupHashes(groupIndex) = inputsHash } @@ -38,17 +36,14 @@ class Evaluator(workspacePath: Path, def evaluateGroupCached(group: OSet[Target[_]], results: collection.Map[Target[_], Any], - groupHashes: collection.Map[Int, Int], - sortedGroups: MultiBiMap[Int, Target[_]]) = { + sortedGroups: MultiBiMap[Int, Target[_]]): (collection.Map[Target[_], Any], Seq[Target[_]]) = { + - pprint.log(group) val (externalInputs, terminals) = partitionGroupInputOutput(group, results) - val upstreamGroupIds = OSet.from(externalInputs.map(sortedGroups.lookupValue), dedup = true) val inputsHash = - externalInputs.toIterator.map(results).hashCode + - group.toIterator.map(_.sideHash).hashCode + - upstreamGroupIds.toIterator.map(groupHashes).hashCode + externalInputs.toIterator.map(results).toVector.hashCode + + group.toIterator.map(_.sideHash).toVector.hashCode() val primeLabel = labeling(terminals.items(0)) @@ -58,33 +53,30 @@ class Evaluator(workspacePath: Path, val cached = for{ json <- util.Try(Json.parse(read.getInputStream(metadataPath))).toOption - (hash, terminalResults) <- Json.fromJson[(Int, Seq[JsValue])](json).asOpt - _ = println("cached hash " + hash) - if hash == inputsHash + (cachedHash, terminalResults) <- Json.fromJson[(Int, Seq[JsValue])](json).asOpt + if cachedHash == inputsHash } yield terminalResults + cached match{ case Some(terminalResults) => - val newResults = mutable.Map.empty[Target[_], Any] + val newResults = mutable.LinkedHashMap.empty[Target[_], Any] for((terminal, res) <- terminals.items.zip(terminalResults)){ newResults(terminal) = terminal.formatter.reads(res).get } - (inputsHash, newResults, Nil) + (newResults, Nil) case _ => - val (newResults, newEvaluated, terminalResults) = { - evaluateGroup(group, results, terminals, targetDestPath) - } - + val (newResults, newEvaluated, terminalResults) = evaluateGroup(group, results, targetDestPath) write.over( metadataPath, Json.prettyPrint( - Json.toJson((inputsHash , terminalResults)) + Json.toJson(inputsHash -> terminals.toList.map(terminalResults)) ), ) - (inputsHash, newResults, newEvaluated) + (newResults, newEvaluated) } } @@ -94,18 +86,17 @@ class Evaluator(workspacePath: Path, val (internalInputs, externalInputs) = allInputs.partition(group.contains) val internalInputSet = internalInputs.toSet val terminals = group.filter(!internalInputSet(_)) - (OSet.from(externalInputs, dedup=true), terminals) + (OSet.from(externalInputs.distinct), terminals) } def evaluateGroup(group: OSet[Target[_]], results: collection.Map[Target[_], Any], - terminals: OSet[Target[_]], targetDestPath: Path) = { rm(targetDestPath) - val terminalResults = mutable.Buffer.empty[JsValue] + val terminalResults = mutable.LinkedHashMap.empty[Target[_], JsValue] val newEvaluated = mutable.Buffer.empty[Target[_]] - val newResults = mutable.Map.empty[Target[_], Any] + val newResults = mutable.LinkedHashMap.empty[Target[_], Any] for (target <- group.items) { newEvaluated.append(target) val targetInputValues = target.inputs.toVector.map(x => @@ -117,9 +108,8 @@ class Evaluator(workspacePath: Path, val (res, serialized) = target.evaluateAndWrite( new Args(targetInputValues, targetDestPath) ) - if (terminals.contains(target)) { - terminalResults.append(serialized) - } + terminalResults(target) = serialized + newResults(target) = res } } diff --git a/src/main/scala/forge/Target.scala b/src/main/scala/forge/Target.scala index f6339e55..fe975638 100644 --- a/src/main/scala/forge/Target.scala +++ b/src/main/scala/forge/Target.scala @@ -55,6 +55,7 @@ object Target{ def evaluate(args: Args) = { counter + args.args.map(_.asInstanceOf[Int]).sum } + override def sideHash = counter } def traverse[T: Format](source: Seq[Target[T]]) = { @@ -78,11 +79,13 @@ object Target{ } def path(path: ammonite.ops.Path) = new Path(path) - class Path(path: ammonite.ops.Path) extends Target[ammonite.ops.Path]{ - def evaluate(args: Args) = path + class Path(path: ammonite.ops.Path) extends Target[PathRef]{ + val handle = PathRef(path) + def evaluate(args: Args) = handle + override def sideHash = handle.hashCode() val inputs = Nil - override def sideHash = ls.rec(path).map(x => (x.toString, x.mtime)).hashCode() } + class Subprocess(val inputs: Seq[Target[_]], command: Args => Seq[String]) extends Target[Subprocess.Result] { @@ -92,10 +95,13 @@ object Target{ implicit val path = ammonite.ops.Path(args.dest, pwd) val output = %%(command(args)) assert(output.exitCode == 0) - Subprocess.Result(output, args.dest) + Subprocess.Result(output, PathRef(args.dest)) } } object Subprocess{ - case class Result(result: ammonite.ops.CommandResult, dest: ammonite.ops.Path) + case class Result(result: ammonite.ops.CommandResult, dest: PathRef) + object Result{ + implicit val tsFormat: Format[Target.Subprocess.Result] = Json.format + } } } diff --git a/src/main/scala/forge/Util.scala b/src/main/scala/forge/Util.scala index 3877015a..c5f71b6b 100644 --- a/src/main/scala/forge/Util.scala +++ b/src/main/scala/forge/Util.scala @@ -1,7 +1,25 @@ package forge +import ammonite.ops.ls +import play.api.libs.json.{Format, Json} + import scala.collection.mutable + +object PathRef{ + implicit def jsonFormatter: Format[PathRef] = Json.format +} +case class PathRef(path: ammonite.ops.Path){ + override def hashCode() = { + if (!path.isDir) path.hashCode() + path.mtime.toMillis.toInt + else ls.rec.iter(path) + .filter(_.isFile) + .map(x => x.toString.hashCode + x.mtime.toMillis) + .sum + .toInt + } +} + trait MultiBiMap[K, V]{ def containsValue(v: V): Boolean def lookupKey(k: K): OSet[V] @@ -57,22 +75,21 @@ trait OSet[V] extends TraversableOnce[V]{ } object OSet{ def apply[V](items: V*) = from(items) - def dedup[V](items: V*) = from(items, dedup = true) - def from[V](items: TraversableOnce[V], dedup: Boolean = false): OSet[V] = { - val set = new MutableOSet[V](dedup) + def from[V](items: TraversableOnce[V]): OSet[V] = { + val set = new MutableOSet[V]() items.foreach(set.append) set } } -class MutableOSet[V](dedup: Boolean = false) extends OSet[V]{ +class MutableOSet[V]() extends OSet[V]{ private[this] val items0 = mutable.ArrayBuffer.empty[V] private[this] val set0 = mutable.Set.empty[V] def contains(v: V) = set0.contains(v) def append(v: V) = if (!contains(v)){ set0.add(v) items0.append(v) - }else if (!dedup) { + }else { throw new Exception("Duplicated item inserted into OrderedSet: " + v) } def appendAll(vs: Seq[V]) = vs.foreach(append) diff --git a/src/main/scala/forge/package.scala b/src/main/scala/forge/package.scala index bcbdc70f..988f7e23 100644 --- a/src/main/scala/forge/package.scala +++ b/src/main/scala/forge/package.scala @@ -41,5 +41,5 @@ package object forge { } implicit val crFormat: Format[ammonite.ops.CommandResult] = Json.format - implicit val tsFormat: Format[Target.Subprocess.Result] = Json.format + } diff --git a/src/test/scala/forge/EvaluationTests.scala b/src/test/scala/forge/EvaluationTests.scala index 7b47646c..2c18be2f 100644 --- a/src/test/scala/forge/EvaluationTests.scala +++ b/src/test/scala/forge/EvaluationTests.scala @@ -22,6 +22,7 @@ object EvaluationTests extends TestSuite{ def apply(target: Target[_], expValue: Any, expEvaled: OSet[Target[_]], extraEvaled: Int = 0) = { + val Evaluator.Results(returnedValues, returnedEvaluated) = evaluator.evaluate(OSet(target)) val (matchingReturnedEvaled, extra) = returnedEvaluated.items.partition(expEvaled.contains) @@ -31,6 +32,7 @@ object EvaluationTests extends TestSuite{ matchingReturnedEvaled.toSet == expEvaled.toSet, extra.length == extraEvaled ) + // Second time the value is already cached, so no evaluation needed val Evaluator.Results(returnedValues2, returnedEvaluated2) = evaluator.evaluate(OSet(target)) assert( diff --git a/src/test/scala/forge/Framework.scala b/src/test/scala/forge/Framework.scala index ccfdf69b..d1aa7434 100644 --- a/src/test/scala/forge/Framework.scala +++ b/src/test/scala/forge/Framework.scala @@ -4,4 +4,8 @@ class Framework extends utest.runner.Framework { override def exceptionStackFrameHighlighter(s: StackTraceElement) = { s.getClassName.startsWith("forge.") } + override def setup() = { + import ammonite.ops._ + rm(pwd / 'target / 'workspace) + } } diff --git a/src/test/scala/forge/Main.scala b/src/test/scala/forge/Main.scala index 99237e32..2ef6083b 100644 --- a/src/test/scala/forge/Main.scala +++ b/src/test/scala/forge/Main.scala @@ -16,38 +16,38 @@ object Main{ val res = evaluator.evaluate(OSet(jar)) println(res.evaluated.collect(mapping)) } - def compileAll(sources: Target[Seq[Path]]) = { + def compileAll(sources: Target[Seq[PathRef]]) = { new Target.Subprocess( Seq(sources), args => Seq("javac") ++ - args[Seq[Path]](0).map(_.toString) ++ + args[Seq[PathRef]](0).map(_.path.toString) ++ Seq("-d", args.dest.toString) ).map(_.dest) } - def list(root: Target[Path]): Target[Seq[Path]] = { - root.map(ls.rec) + def list(root: Target[PathRef]): Target[Seq[PathRef]] = { + root.map(x => ls.rec(x.path).map(PathRef(_))) } - case class jarUp(roots: Target[Path]*) extends Target[Path]{ + case class jarUp(roots: Target[PathRef]*) extends Target[PathRef]{ val inputs = roots - def evaluate(args: Args): Path = { + def evaluate(args: Args): PathRef = { val output = new java.util.jar.JarOutputStream(new FileOutputStream(args.dest.toIO)) for{ root0 <- args.args - root = root0.asInstanceOf[Path] + root = root0.asInstanceOf[PathRef] - path <- ls.rec(root) + path <- ls.rec(root.path) if path.isFile }{ - val relative = path.relativeTo(root) + val relative = path.relativeTo(root.path) output.putNextEntry(new JarEntry(relative.toString)) output.write(read.bytes(path)) } output.close() - args.dest + PathRef(args.dest) } -- cgit v1.2.3