diff options
author | Li Haoyi <haoyi.sg@gmail.com> | 2017-10-29 06:05:59 -0700 |
---|---|---|
committer | Li Haoyi <haoyi.sg@gmail.com> | 2017-10-29 06:05:59 -0700 |
commit | d1a8e26b0b747885782d261d17dbb1ce8d53a36b (patch) | |
tree | dd4c92cea66889f0f649d01dd17566dc1fbb4c7a /src/main | |
parent | c2d3346178790d5b5c4462f55bfb0d23ad2b28a4 (diff) | |
download | mill-d1a8e26b0b747885782d261d17dbb1ce8d53a36b.tar.gz mill-d1a8e26b0b747885782d261d17dbb1ce8d53a36b.tar.bz2 mill-d1a8e26b0b747885782d261d17dbb1ce8d53a36b.zip |
Swap over in-memory results cache to disk-based JSON metadata
Diffstat (limited to 'src/main')
-rw-r--r-- | src/main/scala/forge/Evaluator.scala | 45 | ||||
-rw-r--r-- | src/main/scala/forge/Target.scala | 6 | ||||
-rw-r--r-- | src/main/scala/forge/package.scala | 5 |
3 files changed, 37 insertions, 19 deletions
diff --git a/src/main/scala/forge/Evaluator.scala b/src/main/scala/forge/Evaluator.scala index 8a8c672c..ed63f199 100644 --- a/src/main/scala/forge/Evaluator.scala +++ b/src/main/scala/forge/Evaluator.scala @@ -1,20 +1,16 @@ package forge +import java.nio.charset.Charset import java.nio.{file => jnio} -import play.api.libs.json.Json +import play.api.libs.json.{JsValue, Json} import scala.collection.mutable +import scala.io.Codec class Evaluator(workspacePath: jnio.Path, labeling: Map[Target[_], Seq[String]]){ - /** - * Cache from the ID of the first terminal target in a group to the has of - * all the group's distinct inputs, and the results of the possibly-multiple - * terminal nodes - */ - val resultCache = mutable.Map.empty[String, (Int, Seq[String])] def evaluate(targets: OSet[Target[_]]): Evaluator.Results = { jnio.Files.createDirectories(workspacePath) @@ -40,19 +36,37 @@ class Evaluator(workspacePath: jnio.Path, val (inputsHash, terminals) = partitionGroupInputOutput(group, results) val primeLabel = labeling(terminals.items(0)).mkString("/") - resultCache.get(primeLabel) match{ - case Some((hash, terminalResults)) if hash == inputsHash && !group.exists(_.dirty) => + val targetDestPath = workspacePath.resolve(jnio.Paths.get(primeLabel)) + val metadataPath = targetDestPath.resolveSibling( + targetDestPath.getFileName.toString + ".forge.json" + ) + val cached = for{ + json <- util.Try(Json.parse(jnio.Files.newInputStream(metadataPath))).toOption + (hash, terminalResults) <- Json.fromJson[(Int, Seq[JsValue])](json).asOpt + if hash == inputsHash && !group.exists(_.dirty) + } yield (hash, terminalResults) + + cached match{ + case Some((hash, terminalResults)) => val newResults = mutable.Map.empty[Target[_], Any] for((terminal, res) <- terminals.items.zip(terminalResults)){ - newResults(terminal) = terminal.formatter.reads(Json.parse(res)).get + newResults(terminal) = terminal.formatter.reads(res).get } (newResults, Nil) case _ => val (newResults, newEvaluated, terminalResults) = { - evaluateGroup(group, results, terminals, primeLabel) + evaluateGroup(group, results, terminals, targetDestPath) } - resultCache(primeLabel) = (inputsHash, terminalResults) + + + jnio.Files.write( + metadataPath, + Json.prettyPrint(Json.toJson((inputsHash, terminalResults))).getBytes(Codec.UTF8.charSet), + jnio.StandardOpenOption.CREATE, + jnio.StandardOpenOption.TRUNCATE_EXISTING + ) + (newResults, newEvaluated) } } @@ -71,10 +85,10 @@ class Evaluator(workspacePath: jnio.Path, def evaluateGroup(group: OSet[Target[_]], results: collection.Map[Target[_], Any], terminals: OSet[Target[_]], - primeLabel: String) = { - val targetDestPath = workspacePath.resolve(jnio.Paths.get(primeLabel)) + targetDestPath: jnio.Path) = { + deleteRec(targetDestPath) - val terminalResults = mutable.Buffer.empty[String] + val terminalResults = mutable.Buffer.empty[JsValue] val newEvaluated = mutable.Buffer.empty[Target[_]] val newResults = mutable.Map.empty[Target[_], Any] for (target <- group.items) { @@ -90,7 +104,6 @@ class Evaluator(workspacePath: jnio.Path, ) if (terminals.contains(target)) { terminalResults.append(serialized) - } newResults(target) = res } diff --git a/src/main/scala/forge/Target.scala b/src/main/scala/forge/Target.scala index 8966a729..144fc0e9 100644 --- a/src/main/scala/forge/Target.scala +++ b/src/main/scala/forge/Target.scala @@ -2,7 +2,7 @@ package forge import java.nio.{file => jnio} -import play.api.libs.json.{Format, Json} +import play.api.libs.json.{Format, JsValue, Json} abstract class Target[T](implicit formatter: Format[T]) extends Target.Ops[T]{ /** * What other Targets does this Target depend on? @@ -24,10 +24,10 @@ abstract class Target[T](implicit formatter: Format[T]) extends Target.Ops[T]{ object Target{ abstract class Ops[T](implicit val formatter: Format[T]){ this: Target[T] => - def evaluateAndWrite(args: Args): (T, String) = { + def evaluateAndWrite(args: Args): (T, JsValue) = { val res = evaluate(args) val str = formatter.writes(res) - (res, Json.stringify(str)) + (res, str) } def map[V: Format](f: T => V) = { new Target.Mapped(this, f) diff --git a/src/main/scala/forge/package.scala b/src/main/scala/forge/package.scala index 8145acd3..fe7e1965 100644 --- a/src/main/scala/forge/package.scala +++ b/src/main/scala/forge/package.scala @@ -5,6 +5,11 @@ import ammonite.ops.Bytes package object forge { +// implicit object jsValueFormat extends Format[JsValue]{ +// def reads(json: JsValue) = JsSuccess(json) +// def writes(o: jnio.Path) = JsString(o.toAbsolutePath.toString) +// } + implicit object pathFormat extends Format[jnio.Path]{ def reads(json: JsValue) = json match{ case JsString(v) => JsSuccess(jnio.Paths.get(v)) |