diff options
author | Li Haoyi <haoyi.sg@gmail.com> | 2017-10-29 12:23:14 -0700 |
---|---|---|
committer | Li Haoyi <haoyi.sg@gmail.com> | 2017-10-29 12:23:14 -0700 |
commit | 04a9cac84f0bb6d5f9d573539fbb54c81d689148 (patch) | |
tree | 3c11e75f15123f6944b9613c2e7d6443c40e0fed /src/main | |
parent | e457333ca386fccf3a79c6d69b0eb86d96d47281 (diff) | |
download | mill-04a9cac84f0bb6d5f9d573539fbb54c81d689148.tar.gz mill-04a9cac84f0bb6d5f9d573539fbb54c81d689148.tar.bz2 mill-04a9cac84f0bb6d5f9d573539fbb54c81d689148.zip |
Finished migrating over to using `PathRef` to propagate changes to file paths and folders.
`sideHash` remains, but is only used for direct "do I re-evaluate" checks, and relies on the return values to force downstream targets to re-evaluate if necessary
Diffstat (limited to 'src/main')
-rw-r--r-- | src/main/scala/forge/Evaluator.scala | 52 | ||||
-rw-r--r-- | src/main/scala/forge/Target.scala | 16 | ||||
-rw-r--r-- | src/main/scala/forge/Util.scala | 27 | ||||
-rw-r--r-- | src/main/scala/forge/package.scala | 2 |
4 files changed, 55 insertions, 42 deletions
diff --git a/src/main/scala/forge/Evaluator.scala b/src/main/scala/forge/Evaluator.scala index 85bd5ac6..888f6639 100644 --- a/src/main/scala/forge/Evaluator.scala +++ b/src/main/scala/forge/Evaluator.scala @@ -1,7 +1,7 @@ package forge -import play.api.libs.json.{JsValue, Json} +import play.api.libs.json.{JsValue, Json, Reads} import scala.collection.mutable import ammonite.ops._ @@ -17,19 +17,17 @@ class Evaluator(workspacePath: Path, ) val evaluated = new MutableOSet[Target[_]] - val results = mutable.Map.empty[Target[_], Any] - val groupHashes = mutable.Map.empty[Int, Int] + val results = mutable.LinkedHashMap.empty[Target[_], Any] + for (groupIndex <- sortedGroups.keys()){ val group = sortedGroups.lookupKey(groupIndex) - val (inputsHash, newResults, newEvaluated) = evaluateGroupCached( + val (newResults, newEvaluated) = evaluateGroupCached( group, results, - groupHashes, sortedGroups ) evaluated.appendAll(newEvaluated) for((k, v) <- newResults) results.put(k, v) - groupHashes(groupIndex) = inputsHash } @@ -38,17 +36,14 @@ class Evaluator(workspacePath: Path, def evaluateGroupCached(group: OSet[Target[_]], results: collection.Map[Target[_], Any], - groupHashes: collection.Map[Int, Int], - sortedGroups: MultiBiMap[Int, Target[_]]) = { + sortedGroups: MultiBiMap[Int, Target[_]]): (collection.Map[Target[_], Any], Seq[Target[_]]) = { + - pprint.log(group) val (externalInputs, terminals) = partitionGroupInputOutput(group, results) - val upstreamGroupIds = OSet.from(externalInputs.map(sortedGroups.lookupValue), dedup = true) val inputsHash = - externalInputs.toIterator.map(results).hashCode + - group.toIterator.map(_.sideHash).hashCode + - upstreamGroupIds.toIterator.map(groupHashes).hashCode + externalInputs.toIterator.map(results).toVector.hashCode + + group.toIterator.map(_.sideHash).toVector.hashCode() val primeLabel = labeling(terminals.items(0)) @@ -58,33 +53,30 @@ class Evaluator(workspacePath: Path, val cached = for{ json <- util.Try(Json.parse(read.getInputStream(metadataPath))).toOption - (hash, terminalResults) <- Json.fromJson[(Int, Seq[JsValue])](json).asOpt - _ = println("cached hash " + hash) - if hash == inputsHash + (cachedHash, terminalResults) <- Json.fromJson[(Int, Seq[JsValue])](json).asOpt + if cachedHash == inputsHash } yield terminalResults + cached match{ case Some(terminalResults) => - val newResults = mutable.Map.empty[Target[_], Any] + val newResults = mutable.LinkedHashMap.empty[Target[_], Any] for((terminal, res) <- terminals.items.zip(terminalResults)){ newResults(terminal) = terminal.formatter.reads(res).get } - (inputsHash, newResults, Nil) + (newResults, Nil) case _ => - val (newResults, newEvaluated, terminalResults) = { - evaluateGroup(group, results, terminals, targetDestPath) - } - + val (newResults, newEvaluated, terminalResults) = evaluateGroup(group, results, targetDestPath) write.over( metadataPath, Json.prettyPrint( - Json.toJson((inputsHash , terminalResults)) + Json.toJson(inputsHash -> terminals.toList.map(terminalResults)) ), ) - (inputsHash, newResults, newEvaluated) + (newResults, newEvaluated) } } @@ -94,18 +86,17 @@ class Evaluator(workspacePath: Path, val (internalInputs, externalInputs) = allInputs.partition(group.contains) val internalInputSet = internalInputs.toSet val terminals = group.filter(!internalInputSet(_)) - (OSet.from(externalInputs, dedup=true), terminals) + (OSet.from(externalInputs.distinct), terminals) } def evaluateGroup(group: OSet[Target[_]], results: collection.Map[Target[_], Any], - terminals: OSet[Target[_]], targetDestPath: Path) = { rm(targetDestPath) - val terminalResults = mutable.Buffer.empty[JsValue] + val terminalResults = mutable.LinkedHashMap.empty[Target[_], JsValue] val newEvaluated = mutable.Buffer.empty[Target[_]] - val newResults = mutable.Map.empty[Target[_], Any] + val newResults = mutable.LinkedHashMap.empty[Target[_], Any] for (target <- group.items) { newEvaluated.append(target) val targetInputValues = target.inputs.toVector.map(x => @@ -117,9 +108,8 @@ class Evaluator(workspacePath: Path, val (res, serialized) = target.evaluateAndWrite( new Args(targetInputValues, targetDestPath) ) - if (terminals.contains(target)) { - terminalResults.append(serialized) - } + terminalResults(target) = serialized + newResults(target) = res } } diff --git a/src/main/scala/forge/Target.scala b/src/main/scala/forge/Target.scala index f6339e55..fe975638 100644 --- a/src/main/scala/forge/Target.scala +++ b/src/main/scala/forge/Target.scala @@ -55,6 +55,7 @@ object Target{ def evaluate(args: Args) = { counter + args.args.map(_.asInstanceOf[Int]).sum } + override def sideHash = counter } def traverse[T: Format](source: Seq[Target[T]]) = { @@ -78,11 +79,13 @@ object Target{ } def path(path: ammonite.ops.Path) = new Path(path) - class Path(path: ammonite.ops.Path) extends Target[ammonite.ops.Path]{ - def evaluate(args: Args) = path + class Path(path: ammonite.ops.Path) extends Target[PathRef]{ + val handle = PathRef(path) + def evaluate(args: Args) = handle + override def sideHash = handle.hashCode() val inputs = Nil - override def sideHash = ls.rec(path).map(x => (x.toString, x.mtime)).hashCode() } + class Subprocess(val inputs: Seq[Target[_]], command: Args => Seq[String]) extends Target[Subprocess.Result] { @@ -92,10 +95,13 @@ object Target{ implicit val path = ammonite.ops.Path(args.dest, pwd) val output = %%(command(args)) assert(output.exitCode == 0) - Subprocess.Result(output, args.dest) + Subprocess.Result(output, PathRef(args.dest)) } } object Subprocess{ - case class Result(result: ammonite.ops.CommandResult, dest: ammonite.ops.Path) + case class Result(result: ammonite.ops.CommandResult, dest: PathRef) + object Result{ + implicit val tsFormat: Format[Target.Subprocess.Result] = Json.format + } } } diff --git a/src/main/scala/forge/Util.scala b/src/main/scala/forge/Util.scala index 3877015a..c5f71b6b 100644 --- a/src/main/scala/forge/Util.scala +++ b/src/main/scala/forge/Util.scala @@ -1,7 +1,25 @@ package forge +import ammonite.ops.ls +import play.api.libs.json.{Format, Json} + import scala.collection.mutable + +object PathRef{ + implicit def jsonFormatter: Format[PathRef] = Json.format +} +case class PathRef(path: ammonite.ops.Path){ + override def hashCode() = { + if (!path.isDir) path.hashCode() + path.mtime.toMillis.toInt + else ls.rec.iter(path) + .filter(_.isFile) + .map(x => x.toString.hashCode + x.mtime.toMillis) + .sum + .toInt + } +} + trait MultiBiMap[K, V]{ def containsValue(v: V): Boolean def lookupKey(k: K): OSet[V] @@ -57,22 +75,21 @@ trait OSet[V] extends TraversableOnce[V]{ } object OSet{ def apply[V](items: V*) = from(items) - def dedup[V](items: V*) = from(items, dedup = true) - def from[V](items: TraversableOnce[V], dedup: Boolean = false): OSet[V] = { - val set = new MutableOSet[V](dedup) + def from[V](items: TraversableOnce[V]): OSet[V] = { + val set = new MutableOSet[V]() items.foreach(set.append) set } } -class MutableOSet[V](dedup: Boolean = false) extends OSet[V]{ +class MutableOSet[V]() extends OSet[V]{ private[this] val items0 = mutable.ArrayBuffer.empty[V] private[this] val set0 = mutable.Set.empty[V] def contains(v: V) = set0.contains(v) def append(v: V) = if (!contains(v)){ set0.add(v) items0.append(v) - }else if (!dedup) { + }else { throw new Exception("Duplicated item inserted into OrderedSet: " + v) } def appendAll(vs: Seq[V]) = vs.foreach(append) diff --git a/src/main/scala/forge/package.scala b/src/main/scala/forge/package.scala index bcbdc70f..988f7e23 100644 --- a/src/main/scala/forge/package.scala +++ b/src/main/scala/forge/package.scala @@ -41,5 +41,5 @@ package object forge { } implicit val crFormat: Format[ammonite.ops.CommandResult] = Json.format - implicit val tsFormat: Format[Target.Subprocess.Result] = Json.format + } |