From e958dec0dbbcf7f7a28cd21641e76390fb3dba6a Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Mon, 7 Mar 2016 01:36:40 -0500 Subject: cleanup: whitespace changes, separated more things into their own files, use ++ for strings everywhere. Added ++ method to File and URL and use it in many places --- stage1/ClassPath.scala | 30 +++++++++++++++++ stage1/Stage1.scala | 4 +-- stage1/Stage1Lib.scala | 64 ++++++++++++----------------------- stage1/cbt.scala | 21 ++++++++++++ stage1/classloader.scala | 55 +++++++++++++++---------------- stage1/logger.scala | 2 +- stage1/paths.scala | 22 ++++++------- stage1/resolver.scala | 86 +++++++++++++++++++++++++----------------------- 8 files changed, 158 insertions(+), 126 deletions(-) create mode 100644 stage1/ClassPath.scala create mode 100644 stage1/cbt.scala (limited to 'stage1') diff --git a/stage1/ClassPath.scala b/stage1/ClassPath.scala new file mode 100644 index 0000000..66a1b44 --- /dev/null +++ b/stage1/ClassPath.scala @@ -0,0 +1,30 @@ +package cbt +import java.io._ +import java.net._ +import scala.collection.immutable.Seq + +object ClassPath{ + def apply(files: File*): ClassPath = ClassPath(files.toVector) + def flatten( classPaths: Seq[ClassPath] ): ClassPath = ClassPath( classPaths.map(_.files).flatten ) +} +case class ClassPath(files: Seq[File]){ + private val duplicates = (files diff files.distinct).distinct + assert( + duplicates.isEmpty, + "Duplicate classpath entries found:\n" ++ duplicates.mkString("\n") ++ "\nin classpath:\n"++string + ) + private val nonExisting = files.distinct.filterNot(_.exists) + assert( + duplicates.isEmpty, + "Classpath contains entires that don't exist on disk:\n" ++ nonExisting.mkString("\n") ++ "\nin classpath:\n"++string + ) + + def +:(file: File) = ClassPath(file +: files) + def :+(file: File) = ClassPath(files :+ file) + def ++(other: ClassPath) = ClassPath(files ++ other.files) + def string = strings.mkString( File.pathSeparator ) + def strings = files.map{ + f => f.string ++ ( if(f.isDirectory) "/" else "" ) + } + def toConsole = string +} diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index 8db12cf..1aa3f09 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -63,12 +63,12 @@ abstract class Stage1Base{ val cwd = args(0) val src = stage2.listFiles.toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) - val changeIndicator = new File(stage2Target+"/cbt/Build.class") + val changeIndicator = stage2Target ++ "/cbt/Build.class" logger.stage1("before conditionally running zinc to recompile CBT") if( src.exists(newerThan(_, changeIndicator)) ) { val stage1Classpath = CbtDependency(logger).dependencyClasspath - logger.stage1("cbt.lib has changed. Recompiling with cp: "+stage1Classpath) + logger.stage1("cbt.lib has changed. Recompiling with cp: " ++ stage1Classpath.string) zinc( true, src, stage2Target, stage1Classpath )( zincVersion = "0.3.9", scalaVersion = constants.scalaVersion ) } logger.stage1(s"[$now] calling CbtDependency.classLoader") diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index ac77a92..1ad3030 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -30,35 +30,13 @@ object TrappedExitCode{ } } +case class Context( cwd: File, args: Seq[String], logger: Logger ) -case class Context( cwd: String, args: Seq[String], logger: Logger ) - -case class ClassPath(files: Seq[File]){ - private val duplicates = (files diff files.distinct).distinct - assert( - duplicates.isEmpty, - "Duplicate classpath entries found:\n" + duplicates.mkString("\n") + "\nin classpath:\n"+string - ) - private val nonExisting = files.distinct.filterNot(_.exists) - assert( - duplicates.isEmpty, - "Classpath contains entires that don't exist on disk:\n" + nonExisting.mkString("\n") + "\nin classpath:\n"+string - ) - - def +:(file: File) = ClassPath(file +: files) - def :+(file: File) = ClassPath(files :+ file) - def ++(other: ClassPath) = ClassPath(files ++ other.files) - def string = strings.mkString( File.pathSeparator ) - def strings = files.map{ - f => f.toString + ( if(f.isDirectory) "/" else "" ) - } - def toConsole = string -} -object ClassPath{ - def flatten( classPaths: Seq[ClassPath] ): ClassPath = ClassPath( classPaths.map(_.files).flatten ) +class BaseLib{ + def realpath(name: File) = new File(Paths.get(name.getAbsolutePath).normalize.toString) } -class Stage1Lib( val logger: Logger ){ +class Stage1Lib( val logger: Logger ) extends BaseLib{ lib => // ========== reflection ========== @@ -76,24 +54,24 @@ class Stage1Lib( val logger: Logger ){ } // ========== file system / net ========== - + def array2hex(padTo: Int, array: Array[Byte]): String = { val hex = new java.math.BigInteger(1, array).toString(16) - ("0" * (padTo-hex.size)) + hex + ("0" * (padTo-hex.size)) ++ hex } def md5( bytes: Array[Byte] ): String = array2hex(32, MessageDigest.getInstance("MD5").digest(bytes)) def sha1( bytes: Array[Byte] ): String = array2hex(40, MessageDigest.getInstance("SHA-1").digest(bytes)) - def red(string: String) = scala.Console.RED+string+scala.Console.RESET - def blue(string: String) = scala.Console.BLUE+string+scala.Console.RESET - def green(string: String) = scala.Console.GREEN+string+scala.Console.RESET + def red(string: String) = scala.Console.RED++string++scala.Console.RESET + def blue(string: String) = scala.Console.BLUE++string++scala.Console.RESET + def green(string: String) = scala.Console.GREEN++string++scala.Console.RESET - def download(urlString: URL, target: Path, sha1: Option[String]){ - val incomplete = Paths.get(target+".incomplete"); - if( !Files.exists(target) ){ - new File(target.toString).getParentFile.mkdirs - logger.resolver(blue("downloading ")+urlString) - logger.resolver(blue("to ")+target) + def download(urlString: URL, target: File, sha1: Option[String]){ + val incomplete = Paths.get( target.string ++ ".incomplete" ); + if( !target.exists ){ + target.getParentFile.mkdirs + logger.resolver(blue("downloading ") ++ urlString.string) + logger.resolver(blue("to ") ++ target.string) val stream = urlString.openStream Files.copy(stream, incomplete, StandardCopyOption.REPLACE_EXISTING) sha1.foreach{ @@ -101,10 +79,10 @@ class Stage1Lib( val logger: Logger ){ val expected = hash val actual = this.sha1(Files.readAllBytes(incomplete)) assert( expected == actual, s"$expected == $actual" ) - logger.resolver(green("verified")+" checksum for "+target) + logger.resolver( green("verified") ++ " checksum for " ++ target.string) } stream.close - Files.move(incomplete, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + Files.move(incomplete, Paths.get(target.string), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); } } @@ -123,7 +101,7 @@ class Stage1Lib( val logger: Logger ){ } def runMain(cls: String, args: Seq[String], classLoader: ClassLoader ): ExitCode = { - logger.lib(s"Running $cls.main($args) with classLoader: "+classLoader) + logger.lib(s"Running $cls.main($args) with classLoader: " ++ classLoader.toString) trapExitCode{ classLoader .loadClass(cls) @@ -148,8 +126,8 @@ class Stage1Lib( val logger: Logger ){ )( zincVersion: String, scalaVersion: String ): Unit = { val cp = classpath.string - if(classpath.files.isEmpty) throw new Exception("Trying to compile with empty classpath. Source files: "+files) - if(files.isEmpty) throw new Exception("Trying to compile no files. ClassPath: "+cp) + if(classpath.files.isEmpty) throw new Exception("Trying to compile with empty classpath. Source files: " ++ files.toString) + if(files.isEmpty) throw new Exception("Trying to compile no files. ClassPath: " ++ cp) // only run zinc if files changed, for performance reasons // FIXME: this is broken, need invalidate on changes in dependencies as well @@ -187,7 +165,7 @@ class Stage1Lib( val logger: Logger ){ "-scala-extra", scalaReflect.toString, "-cp", cp, "-d", compileTarget.toString - ) ++ extraArgs.map("-S"+_) ++ files.map(_.toString), + ) ++ extraArgs.map("-S"++_) ++ files.map(_.toString), zinc.classLoader ) } diff --git a/stage1/cbt.scala b/stage1/cbt.scala new file mode 100644 index 0000000..01af0d5 --- /dev/null +++ b/stage1/cbt.scala @@ -0,0 +1,21 @@ +package cbt +import java.io._ +import java.nio.file._ +import java.net._ +object `package`{ + private val lib = new BaseLib + implicit class FileExtensionMethods( file: File ){ + def ++( s: String ): File = { + if(s endsWith "/") throw new Exception( + """Trying to append a String that ends in "/" to a File would loose it. Use .stripSuffix("/") if you need to.""" + ) + new File( file.toString ++ s ) + } + def parent = lib.realpath(file ++ "/..") + def string = file.toString + } + implicit class URLExtensionMethods( url: URL ){ + def ++( s: String ): URL = new URL( url.toString ++ s ) + def string = url.toString + } +} diff --git a/stage1/classloader.scala b/stage1/classloader.scala index 6f2213b..3293dd1 100644 --- a/stage1/classloader.scala +++ b/stage1/classloader.scala @@ -8,25 +8,29 @@ import scala.collection.immutable.Seq object ClassLoaderCache{ private val cache = NailgunLauncher.classLoaderCache - def classLoader( path: String, parent: ClassLoader ): ClassLoader = { - def realpath( name: String ) = Paths.get(new File(name).getAbsolutePath).normalize.toString - val normalized = realpath(path) - if( cache.containsKey(normalized) ){ - //println("FOUND: "+normalized) - cache.get(normalized) + def classLoader( classpath: ClassPath, parent: ClassLoader )(implicit logger: Logger): ClassLoader + = cache.synchronized{ + val lib = new Stage1Lib(logger) + val key = classpath.strings.sorted.mkString(":") + if( cache.containsKey(key) ){ + logger.resolver("CACHE HIT: "++key) + cache.get(key) } else { - //println("PUTTING: "+normalized) - //Try(???).recover{ case e=>e.printStackTrace} - val cl = new cbt.URLClassLoader( ClassPath(Seq(new File(normalized))), parent ) - cache.put( normalized, cl ) + logger.resolver("CACHE MISS: "++key) + val cl = new cbt.URLClassLoader( classpath, parent ) + cache.put( key, cl ) cl } } - def remove( path: String ) = cache.remove( path ) + def remove( classpath: ClassPath ) = { + val key = classpath.strings.sorted.mkString(":") + cache.remove( key ) + } } +/* class MultiClassLoader(parents: Seq[ClassLoader]) extends ClassLoader { override def loadClass(name: String) = { - //System.err.println("LOADING CLASS "+name); + //System.err.println("LOADING CLASS "++name); val c = parents.toStream.map{ parent => Try{ @@ -37,29 +41,24 @@ class MultiClassLoader(parents: Seq[ClassLoader]) extends ClassLoader { }.find(_.isDefined).flatten c.getOrElse( ClassLoader.getSystemClassLoader.loadClass(name) ) } - override def toString = "MultiClassLoader(" + parents.mkString(",") + ")" + override def toString = "MultiClassLoader(" ++ parents.mkString(",") ++ ")" } +*/ case class URLClassLoader(classPath: ClassPath, parent: ClassLoader) extends java.net.URLClassLoader( classPath.strings.map( - path => new URL("file:"+path) + path => new URL("file:"++path) ).toArray, parent ){ override def toString = ( - scala.Console.BLUE + "cbt.URLClassLoader" + scala.Console.RESET + "(\n " + getURLs.map(_.toString).sorted.mkString(",\n ") - + (if(getParent() != ClassLoader.getSystemClassLoader()) ",\n" + getParent().toString.split("\n").map(" "+_).mkString("\n") else "") - + "\n)" + scala.Console.BLUE ++ "cbt.URLClassLoader" ++ scala.Console.RESET + ++ "(\n " ++ getURLs.map(_.toString).sorted.mkString(",\n ") + ++ ( + if(getParent() != ClassLoader.getSystemClassLoader()) + ",\n" ++ getParent().toString.split("\n").map(" "++_).mkString("\n") + else "" + ) + ++ "\n)" ) - import scala.language.existentials - /*override def loadClass(name: String): Class[_] = { - //System.err.println("LOADING CLASS "+name+" in "+this); - try{ - super.loadClass(name) - } catch { - case e: ClassNotFoundException => - // FIXME: Shouldn't this happen automatically? - parent.loadClass(name) - } - }*/ } diff --git a/stage1/logger.scala b/stage1/logger.scala index ecc1579..b1d80a3 100644 --- a/stage1/logger.scala +++ b/stage1/logger.scala @@ -19,7 +19,7 @@ case class Logger(enabledLoggers: Set[String]) { System.err.println( s"[${" "*(6-timeTaken.size)}$timeTaken][$name] $msg" ) } - def showInvocation(method: String, args: Any) = method + "( " + args + " )" + def showInvocation(method: String, args: Any) = method ++ "( " ++ args.toString ++ " )" final def stage1(msg: => String) = logGuarded(names.stage1, msg) final def stage2(msg: => String) = logGuarded(names.stage2, msg) diff --git a/stage1/paths.scala b/stage1/paths.scala index f76c2f7..d3856c8 100644 --- a/stage1/paths.scala +++ b/stage1/paths.scala @@ -1,15 +1,15 @@ package cbt import java.io._ object paths{ - val cbtHome = new File(Option(System.getenv("CBT_HOME")).get) - val mavenCache = new File(cbtHome+"/cache/maven/") - val userHome = new File(Option(System.getProperty("user.home")).get) - val stage1 = new File(Option(System.getenv("STAGE1")).get) - val stage2 = new File(cbtHome + "/stage2/") - val nailgun = new File(Option(System.getenv("NAILGUN")).get) - private val target = Option(System.getenv("TARGET")).get - val stage1Target = new File(stage1 + "/" + target) - val stage2Target = new File(stage2 + "/" + target) - val nailgunTarget = new File(nailgun + "/" + target) - val sonatypeLogin = new File(cbtHome+"/sonatype.login") + val cbtHome: File = new File(Option(System.getenv("CBT_HOME")).get) + val mavenCache: File = cbtHome ++ "/cache/maven" + val userHome: File = new File(Option(System.getProperty("user.home")).get) + val stage1: File = new File(Option(System.getenv("STAGE1")).get) + val stage2: File = cbtHome ++ "/stage2" + val nailgun: File = new File(Option(System.getenv("NAILGUN")).get) + private val target = Option(System.getenv("TARGET")).get.stripSuffix("/") + val stage1Target: File = stage1 ++ ("/" ++ target) + val stage2Target: File = stage2 ++ ("/" ++ target) + val nailgunTarget: File = nailgun ++ ("/" ++ target) + val sonatypeLogin: File = cbtHome ++ "/sonatype.login" } diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 880289c..9b3276b 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -10,7 +10,7 @@ private final class Tree( val root: Dependency, computeChildren: => Seq[Tree] ){ lazy val children = computeChildren def linearize: Seq[Dependency] = root +: children.flatMap(_.linearize) def show(indent: Int = 0): Stream[Char] = { - (" " * indent + root.show + "\n").toStream #::: children.map(_.show(indent+1)).foldLeft(Stream.empty[Char])(_ #::: _) + (" " * indent ++ root.show ++ "\n").toStream #::: children.map(_.show(indent+1)).foldLeft(Stream.empty[Char])(_ #::: _) } } @@ -20,9 +20,11 @@ trait ArtifactInfo extends Dependency{ def version: String protected def str = s"$groupId:$artifactId:$version" - override def show = super.show + s"($str)" + override def show = super.show ++ s"($str)" } abstract class Dependency{ + implicit def logger: Logger + protected def lib = new Stage1Lib(logger) def updated: Boolean //def cacheClassLoader: Boolean = false @@ -49,11 +51,10 @@ abstract class Dependency{ ).par.map(_.exportedClasspath).seq.sortBy(_.string) ) if(cacheDependencyClassLoader){ - val mavenClassPathKey = mavenClassPath.strings.sorted.mkString(":") new URLClassLoader( exportedClasspath ++ buildClassPath, ClassLoaderCache.classLoader( - mavenClassPathKey, new URLClassLoader( mavenClassPath, ClassLoader.getSystemClassLoader ) + mavenClassPath, new URLClassLoader( mavenClassPath, ClassLoader.getSystemClassLoader ) ) ) } else { @@ -85,11 +86,13 @@ abstract class Dependency{ def show: String = this.getClass.getSimpleName // ========== debug ========== def dependencyTree: String = dependencyTreeRecursion() - def logger: Logger - protected def lib = new Stage1Lib(logger) - private def dependencyTreeRecursion(indent: Int = 0): String = ( " " * indent ) + (if(updated) lib.red(show) else show) + dependencies.map(_.dependencyTreeRecursion(indent + 1)).map("\n"+_).mkString("") - - private object cacheDependencyClassLoaderBasicBuild extends Cache[ClassLoader] + private def dependencyTreeRecursion(indent: Int = 0): String = ( + ( " " * indent ) + ++ (if(updated) lib.red(show) else show) + ++ dependencies.map( + _.dependencyTreeRecursion(indent + 1) + ).map( "\n" ++ _.toString ).mkString("") + ) } // TODO: all this hard codes the scala version, needs more flexibility @@ -140,16 +143,18 @@ case class MavenDependency( groupId: String, artifactId: String, version: String def updated = false private val groupPath = groupId.split("\\.").mkString("/") - def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version"+(if(sources) "-sources" else "") + def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version"++(if(sources) "-sources" else "") - private def resolverUrl = if(version.endsWith("-SNAPSHOT")) "https://oss.sonatype.org/content/repositories/snapshots" else "https://repo1.maven.org/maven2" - private def baseUrl = resolverUrl + basePath - private def baseFile = mavenCache + basePath - private def pomFile = baseFile+".pom" - private def jarFile = baseFile+".jar" - //private def coursierJarFile = userHome+"/.coursier/cache/v1/https/repo1.maven.org/maven2"+basePath+".jar" - private def pomUrl = baseUrl+".pom" - private def jarUrl = baseUrl+".jar" + private def resolverUrl:URL = new URL( + if(version.endsWith("-SNAPSHOT")) "https://oss.sonatype.org/content/repositories/snapshots" else "https://repo1.maven.org/maven2" + ) + private def baseUrl: URL = resolverUrl ++ basePath + private def baseFile: File = mavenCache ++ basePath + private def pomFile: File = baseFile ++ ".pom" + private def jarFile: File = baseFile ++ ".jar" + //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar" + private def pomUrl: URL = baseUrl ++ ".pom" + private def jarUrl: URL = baseUrl ++ ".jar" def exportedJars = Seq( jar ) def exportedClasspath = ClassPath( exportedJars ) @@ -157,33 +162,32 @@ case class MavenDependency( groupId: String, artifactId: String, version: String import scala.collection.JavaConversions._ def jarSha1 = { - val file = jarFile+".sha1" - def url = jarUrl+".sha1" + val file = jarFile ++ ".sha1" scala.util.Try{ - lib.download( new URL(url), Paths.get(file), None ) + lib.download( jarUrl ++ ".sha1" , file, None ) // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom - Files.readAllLines(Paths.get(file)).mkString("\n").split(" ").head.trim - }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one + Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim + }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one (not sure that's even possible) } + def pomSha1 = { - val file = pomFile+".sha1" - def url = pomUrl+".sha1" + val file = pomFile++".sha1" scala.util.Try{ - lib.download( new URL(url), Paths.get(file), None ) + lib.download( pomUrl++".sha1" , file, None ) // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom - Files.readAllLines(Paths.get(file)).mkString("\n").split(" ").head.trim - }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one + Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim + }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one (not sure that's even possible) } + def jar = { - lib.download( new URL(jarUrl), Paths.get(jarFile), jarSha1 ) - new File(jarFile) - } - def pomXml = { - XML.loadFile(pom.toString) + lib.download( jarUrl, jarFile, jarSha1 ) + jarFile } + def pomXml = XML.loadFile(pom.toString) + def pom = { - lib.download( new URL(pomUrl), Paths.get(pomFile), pomSha1 ) - new File(pomFile) + lib.download( pomUrl, pomFile, pomSha1 ) + pomFile } // ========== pom traversal ========== @@ -211,13 +215,13 @@ case class MavenDependency( groupId: String, artifactId: String, version: String }.toVector } def lookup( xml: Node, accessor: Node => NodeSeq ): Option[String] = { - //println("lookup in "+pomUrl) - val Substitution = "\\$\\{([a-z0-9\\.]+)\\}".r + //println("lookup in "++pomUrl) + val Substitution = "\\$\\{([a-z0-9\\.]++)\\}".r accessor(xml).headOption.flatMap{v => - //println("found: "+v.text) + //println("found: "++v.text) v.text match { case Substitution(path) => - //println("lookup "+path + ": "+(pomXml\path).text) + //println("lookup "++path ++ ": "++(pomXml\path).text) lookup(pomXml, _ \ "properties" \ path) case value => Option(value) } @@ -247,7 +251,7 @@ object MavenDependency{ def removeOutdated( deps: Seq[ArtifactInfo], versionLessThan: (String, String) => Boolean = semanticVersionLessThan - ): Seq[ArtifactInfo] = { + )(implicit logger: Logger): Seq[ArtifactInfo] = { val latest = deps .groupBy( d => (d.groupId, d.artifactId) ) .mapValues( @@ -257,7 +261,7 @@ object MavenDependency{ deps.flatMap{ d => val l = latest.get((d.groupId,d.artifactId)) - //if(d != l) println("EVICTED: "+d.show) + if(d != l) logger.resolver("outdated: "++d.show) l }.distinct } -- cgit v1.2.3