diff options
author | Li Haoyi <haoyi.sg@gmail.com> | 2018-02-25 11:35:28 -0800 |
---|---|---|
committer | Li Haoyi <haoyi.sg@gmail.com> | 2018-02-25 11:35:28 -0800 |
commit | 63f72399537331d30a32d1e71e238c6e97dabc3e (patch) | |
tree | a8d1470335ddb3c1b69adddcf7b1be4c1d00d9fc /core | |
parent | 677a70fc1a55f81b8b1e51bcbb6d698648e644fa (diff) | |
download | mill-63f72399537331d30a32d1e71e238c6e97dabc3e.tar.gz mill-63f72399537331d30a32d1e71e238c6e97dabc3e.tar.bz2 mill-63f72399537331d30a32d1e71e238c6e97dabc3e.zip |
Save the signature of a `PathRef` to avoid re-stating files every time they are de-serialized
Diffstat (limited to 'core')
-rw-r--r-- | core/src/mill/eval/Evaluator.scala | 28 | ||||
-rw-r--r-- | core/src/mill/eval/PathRef.scala | 83 |
2 files changed, 64 insertions, 47 deletions
diff --git a/core/src/mill/eval/Evaluator.scala b/core/src/mill/eval/Evaluator.scala index fd9d6bbe..45a70f82 100644 --- a/core/src/mill/eval/Evaluator.scala +++ b/core/src/mill/eval/Evaluator.scala @@ -10,6 +10,7 @@ import mill.eval.Result.OuterStack import mill.util import mill.util._ import mill.util.Strict.Agg +import upickle.Js import scala.collection.mutable import scala.util.control.NonFatal @@ -141,10 +142,13 @@ case class Evaluator[T](outPath: Path, mkdir(paths.out) val cached = for{ json <- scala.util.Try(upickle.json.read(read(paths.meta))).toOption - (cachedHash, terminalResult) <- scala.util.Try(upickle.default.readJs[(Int, upickle.Js.Value)](json)).toOption - if cachedHash == inputsHash + obj <- scala.util.Try(upickle.default.readJs[Js.Obj](json).value.toMap).toOption + cacheInputsHash <- obj.get("inputsHash").map(_.num) + cacheValueHash <- obj.get("valueHash").map(_.num) + cacheValue <- obj.get("value") + if cacheInputsHash == inputsHash reader <- labelledNamedTask.format - parsed <- reader.read.lift(terminalResult) + parsed <- reader.read.lift(cacheValue) } yield parsed val workerCached = labelledNamedTask.task.asWorker @@ -186,10 +190,20 @@ case class Evaluator[T](outPath: Path, val terminalResult = labelledNamedTask .writer .asInstanceOf[Option[upickle.default.Writer[Any]]] - .map(_.write(v)) - - for(t <- terminalResult){ - write.over(paths.meta, upickle.default.write(inputsHash -> t, indent = 4)) + .map(_.write(v) -> v) + + for((json, v) <- terminalResult){ + write.over( + paths.meta, + upickle.default.write( + Js.Obj( + "inputsHash" -> Js.Num(inputsHash), + "valueHash" -> Js.Num(v.hashCode()), + "value" -> json + ), + indent = 4 + ) + ) } } diff --git a/core/src/mill/eval/PathRef.scala b/core/src/mill/eval/PathRef.scala index 0a629a14..168eaa9a 100644 --- a/core/src/mill/eval/PathRef.scala +++ b/core/src/mill/eval/PathRef.scala @@ -15,56 +15,59 @@ import mill.util.JsonFormatters * on the contents of the filesystem underneath it. Used to ensure filesystem * changes can bust caches which are keyed off hashcodes. */ -case class PathRef(path: ammonite.ops.Path, quick: Boolean = false){ - val sig = { - val digest = MessageDigest.getInstance("MD5") +case class PathRef(path: ammonite.ops.Path, quick: Boolean, sig: Int){ + override def hashCode() = sig +} - val buffer = new Array[Byte](16 * 1024) - jnio.Files.walkFileTree( - path.toNIO, - new FileVisitor[jnio.Path] { - def preVisitDirectory(dir: jnio.Path, attrs: BasicFileAttributes) = { - digest.update(dir.toAbsolutePath.toString.getBytes) - FileVisitResult.CONTINUE - } +object PathRef{ + def apply(path: ammonite.ops.Path, quick: Boolean = false) = { + val sig = { + val digest = MessageDigest.getInstance("MD5") - def visitFile(file: jnio.Path, attrs: BasicFileAttributes) = { - digest.update(file.toAbsolutePath.toString.getBytes) - if (quick){ - val value = (path.mtime.toMillis, path.size).hashCode() - digest.update((value >>> 24).toByte) - digest.update((value >>> 16).toByte) - digest.update((value >>> 8).toByte) - digest.update(value.toByte) - }else { - val is = jnio.Files.newInputStream(file) + val buffer = new Array[Byte](16 * 1024) + jnio.Files.walkFileTree( + path.toNIO, + new FileVisitor[jnio.Path] { + def preVisitDirectory(dir: jnio.Path, attrs: BasicFileAttributes) = { + digest.update(dir.toAbsolutePath.toString.getBytes) + FileVisitResult.CONTINUE + } + + def visitFile(file: jnio.Path, attrs: BasicFileAttributes) = { + digest.update(file.toAbsolutePath.toString.getBytes) + if (quick){ + val value = (path.mtime.toMillis, path.size).hashCode() + digest.update((value >>> 24).toByte) + digest.update((value >>> 16).toByte) + digest.update((value >>> 8).toByte) + digest.update(value.toByte) + }else { + val is = jnio.Files.newInputStream(file) - def rec(): Unit = { - val length = is.read(buffer) - if (length != -1) { - digest.update(buffer, 0, length) - rec() + def rec(): Unit = { + val length = is.read(buffer) + if (length != -1) { + digest.update(buffer, 0, length) + rec() + } } - } - rec() + rec() - is.close() + is.close() + } + FileVisitResult.CONTINUE } - FileVisitResult.CONTINUE - } - def visitFileFailed(file: jnio.Path, exc: IOException) = FileVisitResult.CONTINUE - def postVisitDirectory(dir: jnio.Path, exc: IOException) = FileVisitResult.CONTINUE - } - ) + def visitFileFailed(file: jnio.Path, exc: IOException) = FileVisitResult.CONTINUE + def postVisitDirectory(dir: jnio.Path, exc: IOException) = FileVisitResult.CONTINUE + } + ) - java.util.Arrays.hashCode(digest.digest()) + java.util.Arrays.hashCode(digest.digest()) + } + new PathRef(path, quick, sig) } - override def hashCode() = sig -} - -object PathRef{ private implicit val pathFormat: RW[Path] = JsonFormatters.pathReadWrite implicit def jsonFormatter: RW[PathRef] = upickle.default.macroRW } |