diff options
author | gehnaphore <geh@georgeandjulia.com> | 2018-12-19 23:41:31 -0800 |
---|---|---|
committer | Tobias Roeser <le.petit.fou@web.de> | 2018-12-20 08:41:31 +0100 |
commit | 056cd88f77d09ad7237933a38d441862501d8739 (patch) | |
tree | f5d3a2ac74476d11472fffcd98f678c6b6a322b7 | |
parent | 2752dcffae3765c56dca98b11a6a32318530e961 (diff) | |
download | mill-056cd88f77d09ad7237933a38d441862501d8739.tar.gz mill-056cd88f77d09ad7237933a38d441862501d8739.tar.bz2 mill-056cd88f77d09ad7237933a38d441862501d8739.zip |
Avoid unnecessary dependency downloading by providing fetches per cache policy (#494)
* Avoid unnecessary dependency downloading by providing fetches per cache policy; add ticker logging when they are downloading
* Fix GenIdeaTests by making the Log context Option[]al
* Add some comments
* Rebase and resolve
-rw-r--r-- | main/core/src/eval/Evaluator.scala | 12 | ||||
-rw-r--r-- | main/core/src/util/Loggers.scala | 18 | ||||
-rw-r--r-- | main/src/modules/Jvm.scala | 79 | ||||
-rw-r--r-- | scalajslib/src/ScalaJSModule.scala | 3 | ||||
-rw-r--r-- | scalalib/src/GenIdeaImpl.scala | 8 | ||||
-rw-r--r-- | scalalib/src/JavaModule.scala | 3 | ||||
-rw-r--r-- | scalalib/src/Lib.scala | 12 | ||||
-rw-r--r-- | scalalib/src/ZincWorkerModule.scala | 3 | ||||
-rw-r--r-- | scalalib/test/src/GenIdeaTests.scala | 2 | ||||
-rw-r--r-- | scalanativelib/src/ScalaNativeModule.scala | 9 |
10 files changed, 128 insertions, 21 deletions
diff --git a/main/core/src/eval/Evaluator.scala b/main/core/src/eval/Evaluator.scala index dbaf9433..71fe85a9 100644 --- a/main/core/src/eval/Evaluator.scala +++ b/main/core/src/eval/Evaluator.scala @@ -251,7 +251,7 @@ case class Evaluator(home: os.Path, val nonEvaluatedTargets = group.indexed.filterNot(results.contains) - maybeTargetLabel.foreach { targetLabel => + val tickerPrefix = maybeTargetLabel.map { targetLabel => val inputResults = for { target <- nonEvaluatedTargets item <- target.inputs.filterNot(group.contains) @@ -259,10 +259,16 @@ case class Evaluator(home: os.Path, val logRun = inputResults.forall(_.isInstanceOf[Result.Success[_]]) - if(logRun) { log.ticker(s"[$counterMsg] $targetLabel ") } + val prefix = s"[$counterMsg] $targetLabel " + if(logRun) log.ticker(prefix) + prefix + "| " } - val multiLogger = resolveLogger(paths.map(_.log)) + val multiLogger = new ProxyLogger(resolveLogger(paths.map(_.log))) { + override def ticker(s: String): Unit = { + super.ticker(tickerPrefix.getOrElse("")+s) + } + } var usedDest = Option.empty[(Task[_], Array[StackTraceElement])] for (task <- nonEvaluatedTargets) { newEvaluated.append(task) diff --git a/main/core/src/util/Loggers.scala b/main/core/src/util/Loggers.scala index aab1a324..4771d782 100644 --- a/main/core/src/util/Loggers.scala +++ b/main/core/src/util/Loggers.scala @@ -188,3 +188,21 @@ case class MultiLogger(colored: Boolean, logger1: Logger, logger2: Logger) exten logger2.close() } } + +/** + * A Logger that forwards all logging to another Logger. Intended to be + * used as a base class for wrappers that modify logging behavior. + */ +case class ProxyLogger(logger: Logger) extends Logger { + def colored = logger.colored + + lazy val outputStream = logger.outputStream + lazy val errorStream = logger.errorStream + lazy val inStream = logger.inStream + + def info(s: String) = logger.info(s) + def error(s: String) = logger.error(s) + def ticker(s: String) = logger.ticker(s) + def debug(s: String) = logger.debug(s) + override def close() = logger.close() +} diff --git a/main/src/modules/Jvm.scala b/main/src/modules/Jvm.scala index e17631e3..020de4b1 100644 --- a/main/src/modules/Jvm.scala +++ b/main/src/modules/Jvm.scala @@ -8,7 +8,7 @@ import java.nio.file.attribute.PosixFilePermission import java.util.Collections import java.util.jar.{JarEntry, JarFile, JarOutputStream} -import coursier.{Cache, Dependency, Fetch, Repository, Resolution} +import coursier.{Cache, Dependency, Fetch, Repository, Resolution, CachePolicy} import coursier.util.{Gather, Task} import geny.Generator import mill.main.client.InputPumper @@ -402,10 +402,11 @@ object Jvm { deps: TraversableOnce[coursier.Dependency], force: TraversableOnce[coursier.Dependency], sources: Boolean = false, - mapDependencies: Option[Dependency => Dependency] = None): Result[Agg[PathRef]] = { + mapDependencies: Option[Dependency => Dependency] = None, + ctx: Option[mill.util.Ctx.Log] = None): Result[Agg[PathRef]] = { val (_, resolution) = resolveDependenciesMetadata( - repositories, deps, force, mapDependencies + repositories, deps, force, mapDependencies, ctx ) val errs = resolution.metadataErrors if(errs.nonEmpty) { @@ -459,7 +460,10 @@ object Jvm { def resolveDependenciesMetadata(repositories: Seq[Repository], deps: TraversableOnce[coursier.Dependency], force: TraversableOnce[coursier.Dependency], - mapDependencies: Option[Dependency => Dependency] = None) = { + mapDependencies: Option[Dependency => Dependency] = None, + ctx: Option[mill.util.Ctx.Log] = None) = { + + val cachePolicies = CachePolicy.default val forceVersions = force .map(mapDependencies.getOrElse(identity[Dependency](_))) @@ -472,10 +476,75 @@ object Jvm { mapDependencies = mapDependencies ) - val fetch = Fetch.from(repositories, Cache.fetch[Task]()) + val resolutionLogger = ctx.map(c => new TickerResolutionLogger(c)) + val fetches = cachePolicies.map { p => + Cache.fetch[Task]( + logger = resolutionLogger, + cachePolicy = p + ) + } + + val fetch = Fetch.from(repositories, fetches.head, fetches.tail: _*) import scala.concurrent.ExecutionContext.Implicits.global val resolution = start.process.run(fetch).unsafeRun() (deps.toSeq, resolution) } + + /** + * A Coursier Cache.Logger implementation that updates the ticker with the count and + * overall byte size of artifacts being downloaded. + * + * In practice, this ticket output gets prefixed with the current target for which + * dependencies are being resolved, using a ProxyLogger subclass. + */ + class TickerResolutionLogger(ctx: mill.util.Ctx.Log) extends Cache.Logger { + case class DownloadState(var current: Long, var total: Long) + var downloads = new mutable.TreeMap[String,DownloadState]() + var totalDownloadCount = 0 + var finishedCount = 0 + var finishedState = DownloadState(0,0) + + def updateTicker(): Unit = { + val sums = downloads.values + .fold(DownloadState(0,0)) { + (s1, s2) => DownloadState( + s1.current + s2.current, + Math.max(s1.current,s1.total) + Math.max(s2.current,s2.total) + ) + } + sums.current += finishedState.current + sums.total += finishedState.total + ctx.log.ticker(s"Downloading [${downloads.size + finishedCount}/$totalDownloadCount] artifacts (~${sums.current}/${sums.total} bytes)") + } + + override def downloadingArtifact(url: String, file: File): Unit = synchronized { + totalDownloadCount += 1 + downloads += url -> DownloadState(0,0) + updateTicker() + } + + override def downloadLength(url: String, totalLength: Long, alreadyDownloaded: Long, watching: Boolean): Unit = synchronized { + val state = downloads(url) + state.current = alreadyDownloaded + state.total = totalLength + updateTicker() + } + + override def downloadProgress(url: String, downloaded: Long): Unit = synchronized { + val state = downloads(url) + state.current = downloaded + updateTicker() + } + + override def downloadedArtifact(url: String, success: Boolean): Unit = synchronized { + val state = downloads(url) + finishedState.current += state.current + finishedState.total += Math.max(state.current, state.total) + finishedCount += 1 + downloads -= url + updateTicker() + } + } + } diff --git a/scalajslib/src/ScalaJSModule.scala b/scalajslib/src/ScalaJSModule.scala index 137e8ee2..230a3e5c 100644 --- a/scalajslib/src/ScalaJSModule.scala +++ b/scalajslib/src/ScalaJSModule.scala @@ -48,7 +48,8 @@ trait ScalaJSModule extends scalalib.ScalaModule { outer => resolveDependencies( repositories, Lib.depToDependency(_, "2.12.4", ""), - commonDeps :+ envDep + commonDeps :+ envDep, + ctx = Some(implicitly[mill.util.Ctx.Log]) ) } diff --git a/scalalib/src/GenIdeaImpl.scala b/scalalib/src/GenIdeaImpl.scala index b8f9d35e..261a83a9 100644 --- a/scalalib/src/GenIdeaImpl.scala +++ b/scalalib/src/GenIdeaImpl.scala @@ -42,7 +42,7 @@ object GenIdeaImpl { val evaluator = new Evaluator(ctx.home, os.pwd / 'out, os.pwd / 'out, rootModule, ctx.log) - for((relPath, xml) <- xmlFileLayout(evaluator, rootModule, jdkInfo)){ + for((relPath, xml) <- xmlFileLayout(evaluator, rootModule, jdkInfo, Some(ctx))){ os.write.over(os.pwd/relPath, pp.format(xml), createFolders = true) } } @@ -61,6 +61,7 @@ object GenIdeaImpl { def xmlFileLayout(evaluator: Evaluator, rootModule: mill.Module, jdkInfo: (String,String), + ctx: Option[Log], fetchMillModules: Boolean = true): Seq[(os.RelPath, scala.xml.Node)] = { val modules = rootModule.millInternal.segmentsToModules.values @@ -78,7 +79,10 @@ object GenIdeaImpl { repos.toList, Lib.depToDependency(_, "2.12.4", ""), for(name <- artifactNames) - yield ivy"com.lihaoyi::mill-$name:${sys.props("MILL_VERSION")}" + yield ivy"com.lihaoyi::mill-$name:${sys.props("MILL_VERSION")}", + false, + None, + ctx ) res.items.toList.map(_.path) } diff --git a/scalalib/src/JavaModule.scala b/scalalib/src/JavaModule.scala index 72c0a5a6..1a5bc47b 100644 --- a/scalalib/src/JavaModule.scala +++ b/scalalib/src/JavaModule.scala @@ -138,7 +138,8 @@ trait JavaModule extends mill.Module with TaskModule { outer => resolveCoursierDependency().apply(_), deps(), sources, - mapDependencies = Some(mapDependencies()) + mapDependencies = Some(mapDependencies()), + Some(implicitly[mill.util.Ctx.Log]) ) } diff --git a/scalalib/src/Lib.scala b/scalalib/src/Lib.scala index b8b253bd..bd535efd 100644 --- a/scalalib/src/Lib.scala +++ b/scalalib/src/Lib.scala @@ -35,13 +35,15 @@ object Lib{ def resolveDependenciesMetadata(repositories: Seq[Repository], depToDependency: Dep => coursier.Dependency, deps: TraversableOnce[Dep], - mapDependencies: Option[Dependency => Dependency] = None) = { + mapDependencies: Option[Dependency => Dependency] = None, + ctx: Option[mill.util.Ctx.Log] = None) = { val depSeq = deps.toSeq mill.modules.Jvm.resolveDependenciesMetadata( repositories, depSeq.map(depToDependency), depSeq.filter(_.force).map(depToDependency), - mapDependencies + mapDependencies, + ctx ) } /** @@ -55,14 +57,16 @@ object Lib{ depToDependency: Dep => coursier.Dependency, deps: TraversableOnce[Dep], sources: Boolean = false, - mapDependencies: Option[Dependency => Dependency] = None): Result[Agg[PathRef]] = { + mapDependencies: Option[Dependency => Dependency] = None, + ctx: Option[mill.util.Ctx.Log] = None): Result[Agg[PathRef]] = { val depSeq = deps.toSeq mill.modules.Jvm.resolveDependencies( repositories, depSeq.map(depToDependency), depSeq.filter(_.force).map(depToDependency), sources, - mapDependencies + mapDependencies, + ctx ) } def scalaCompilerIvyDeps(scalaOrganization: String, scalaVersion: String) = diff --git a/scalalib/src/ZincWorkerModule.scala b/scalalib/src/ZincWorkerModule.scala index 97d84aaf..f0474760 100644 --- a/scalalib/src/ZincWorkerModule.scala +++ b/scalalib/src/ZincWorkerModule.scala @@ -102,7 +102,8 @@ trait ZincWorkerModule extends mill.Module{ resolveDependencies( repositories, Lib.depToDependency(_, "2.12.4", ""), - Seq(ivy"org.scala-sbt:compiler-interface:${Versions.zinc}") + Seq(ivy"org.scala-sbt:compiler-interface:${Versions.zinc}"), + ctx = Some(implicitly[mill.util.Ctx.Log]) ) } diff --git a/scalalib/test/src/GenIdeaTests.scala b/scalalib/test/src/GenIdeaTests.scala index 50db95c0..f8d9a0ed 100644 --- a/scalalib/test/src/GenIdeaTests.scala +++ b/scalalib/test/src/GenIdeaTests.scala @@ -28,7 +28,7 @@ object GenIdeaTests extends TestSuite { val layout = GenIdeaImpl.xmlFileLayout( helloWorldEvaluator.evaluator, HelloWorld, - ("JDK_1_8", "1.8 (1)"), fetchMillModules = false) + ("JDK_1_8", "1.8 (1)"), None, fetchMillModules = false) for((relPath, xml) <- layout){ os.write.over(millSourcePath/ "generated"/ relPath, pp.format(xml), createFolders = true) } diff --git a/scalanativelib/src/ScalaNativeModule.scala b/scalanativelib/src/ScalaNativeModule.scala index 289ba759..38525032 100644 --- a/scalanativelib/src/ScalaNativeModule.scala +++ b/scalanativelib/src/ScalaNativeModule.scala @@ -52,7 +52,8 @@ trait ScalaNativeModule extends ScalaModule { outer => Lib.resolveDependencies( Seq(Cache.ivy2Local, MavenRepository("https://repo1.maven.org/maven2")), Lib.depToDependency(_, "2.12.4", ""), - Seq(ivy"com.lihaoyi::mill-scalanativelib-worker-${scalaNativeBinaryVersion()}:${sys.props("MILL_VERSION")}") + Seq(ivy"com.lihaoyi::mill-scalanativelib-worker-${scalaNativeBinaryVersion()}:${sys.props("MILL_VERSION")}"), + ctx = Some(implicitly[mill.util.Ctx.Log]) ) } @@ -83,7 +84,8 @@ trait ScalaNativeModule extends ScalaModule { outer => Lib.resolveDependencies( Seq(Cache.ivy2Local, MavenRepository("https://repo1.maven.org/maven2")), Lib.depToDependency(_, scalaVersion(), platformSuffix()), - toolsIvyDeps() + toolsIvyDeps(), + ctx = Some(implicitly[mill.util.Ctx.Log]) ).map(t => (scalaNativeWorkerClasspath().toSeq ++ t.toSeq).map(_.path)) } @@ -200,7 +202,8 @@ trait TestScalaNativeModule extends ScalaNativeModule with TestModule { testOute Lib.resolveDependencies( repositories, Lib.depToDependency(_, scalaVersion(), ""), - transitiveIvyDeps().filter(d => d.cross.isBinary && supportedTestFrameworks(d.dep.module.name)) + transitiveIvyDeps().filter(d => d.cross.isBinary && supportedTestFrameworks(d.dep.module.name)), + ctx = Some(implicitly[mill.util.Ctx.Log]) ) } |