Merge pull request #1636 from dotty-staging/topic/restructure-attempt2

Restructuring repo and issues

1 files changed, 224 insertions, 66 deletions

diff --git a/project/Build.scala b/project/Build.scala
index 5b8465b4e..699c7ea8a 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -1,5 +1,6 @@
 import sbt.Keys._
 import sbt._
+import complete.DefaultParsers._
 import java.io.{ RandomAccessFile, File }
 import java.nio.channels.FileLock
 import scala.reflect.io.Path
@@ -13,7 +14,7 @@ object DottyBuild extends Build {
   val baseVersion = "0.1"
   val isNightly = sys.env.get("NIGHTLYBUILD") == Some("yes")
 
-  val jenkinsMemLimit = List("-Xmx1300m")
+  val jenkinsMemLimit = List("-Xmx1500m")
 
   val JENKINS_BUILD = "dotty.jenkins.build"
 
@@ -26,6 +27,21 @@ object DottyBuild extends Build {
     // "-XX:+HeapDumpOnOutOfMemoryError", "-Xmx1g", "-Xss2m"
   )
 
+  // Packages all subprojects to their jars
+  lazy val packageAll =
+    taskKey[Map[String, String]]("Package everything needed to run tests")
+
+  // Spawns a repl with the correct classpath
+  lazy val repl = inputKey[Unit]("run the REPL with correct classpath")
+
+  // Used to compile files similar to ./bin/dotc script
+  lazy val dotc =
+    inputKey[Unit]("run the compiler using the correct classpath, or the user supplied classpath")
+
+  // Used to run binaries similar to ./bin/dotr script
+  lazy val dotr =
+    inputKey[Unit]("run compiled binary using the correct classpath, or the user supplied classpath")
+
   override def settings: Seq[Setting[_]] = {
     super.settings ++ Seq(
       scalaVersion in Global := "2.11.5",
@@ -41,7 +57,12 @@ object DottyBuild extends Build {
       homepage in Global := Some(url("https://github.com/lampepfl/dotty")),
 
       // scalac options
-      scalacOptions in Global ++= Seq("-feature", "-deprecation", "-encoding", "UTF8", "-language:existentials,higherKinds,implicitConversions"),
+      scalacOptions in Global ++= Seq(
+        "-feature",
+        "-deprecation",
+        "-encoding", "UTF8",
+        "-language:existentials,higherKinds,implicitConversions"
+      ),
 
       javacOptions in Global ++= Seq("-Xlint:unchecked", "-Xlint:deprecation")
     )
@@ -51,7 +72,57 @@ object DottyBuild extends Build {
   private val overrideScalaVersionSetting =
     ivyScala := ivyScala.value.map(_.copy(overrideScalaVersion = true))
 
+  // set sources to src/, tests to test/ and resources to resources/
+  lazy val sourceStructure = Seq(
+    scalaSource       in Compile    := baseDirectory.value / "src",
+    scalaSource       in Test       := baseDirectory.value / "test",
+    javaSource        in Compile    := baseDirectory.value / "src",
+    javaSource        in Test       := baseDirectory.value / "test",
+    resourceDirectory in Compile    := baseDirectory.value / "resources"
+  )
+
+
+  /** Projects -------------------------------------------------------------- */
+  // The root project:
+  // - aggregates other projects so that "compile", "test", etc are run on all projects at once.
+  // - publishes its own empty artifact "dotty" that depends on "dotty-library" and "dotty-compiler",
+  //   this is only necessary for compatibility with sbt which currently hardcodes the "dotty" artifact name
+  lazy val dotty = project.in(file(".")).
+    // FIXME: we do not aggregate `bin` because its tests delete jars, thus breaking other tests
+    aggregate(`dotty-interfaces`, `dotty-library`, `dotty-compiler`, `dotty-sbt-bridge`, `scala-library`).
+    dependsOn(`dotty-compiler`).
+    dependsOn(`dotty-library`).
+    settings(
+      addCommandAlias("run", "dotty-compiler/run") ++
+      addCommandAlias(
+        "partest",
+        ";packageAll" +
+        ";dotty-compiler/test:runMain dotc.build" +
+        ";dotty-compiler/lockPartestFile" +
+        ";dotty-compiler/test:test" +
+        ";dotty-compiler/runPartestRunner"
+      ) ++
+      addCommandAlias(
+        "partest-only",
+        ";packageAll" +
+        ";dotty-compiler/test:runMain dotc.build" +
+        ";dotty-compiler/lockPartestFile" +
+        ";dotty-compiler/test:test-only dotc.tests" +
+        ";dotty-compiler/runPartestRunner"
+      ) ++
+      addCommandAlias(
+        "partest-only-no-bootstrap",
+        ";packageAll" +
+        ";dotty-compiler/lockPartestFile" +
+        ";dotty-compiler/test:test-only dotc.tests" +
+        ";dotty-compiler/runPartestRunner"
+      )
+    ).
+    settings(publishing)
+
+
   lazy val `dotty-interfaces` = project.in(file("interfaces")).
+    settings(sourceStructure).
     settings(
       // Do not append Scala versions to the generated artifacts
       crossPaths := false,
@@ -62,21 +133,19 @@ object DottyBuild extends Build {
     ).
     settings(publishing)
 
-  lazy val dotty = project.in(file(".")).
+  lazy val `dotty-compiler` = project.in(file("compiler")).
     dependsOn(`dotty-interfaces`).
+    dependsOn(`dotty-library`).
+    settings(sourceStructure).
     settings(
       overrideScalaVersionSetting,
 
-      // set sources to src/, tests to test/ and resources to resources/
-      scalaSource in Compile := baseDirectory.value / "src",
-      javaSource in Compile := baseDirectory.value / "src",
-      scalaSource in Test := baseDirectory.value / "test",
-      javaSource in Test := baseDirectory.value / "test",
-      resourceDirectory in Compile := baseDirectory.value / "resources",
+      // necessary evil: dottydoc currently needs to be included in the dotty
+      // project, for sbt integration
       unmanagedSourceDirectories in Compile := Seq((scalaSource in Compile).value),
-      unmanagedSourceDirectories in Compile += baseDirectory.value / "dottydoc" / "src",
+      unmanagedSourceDirectories in Compile += baseDirectory.value / ".." / "doc-tool" / "src",
       unmanagedSourceDirectories in Test := Seq((scalaSource in Test).value),
-      unmanagedSourceDirectories in Test += baseDirectory.value / "dottydoc" / "test",
+      unmanagedSourceDirectories in Test += baseDirectory.value / ".." / "doc-tool" / "test",
 
       // set system in/out for repl
       connectInput in run := true,
@@ -108,8 +177,70 @@ object DottyBuild extends Build {
       // enable improved incremental compilation algorithm
       incOptions := incOptions.value.withNameHashing(true),
 
+      // packageAll packages all and then returns a map with the abs location
+      packageAll := {
+        Map(
+          "dotty-interfaces" -> (packageBin in (`dotty-interfaces`, Compile)).value,
+          "dotty-compiler" -> (packageBin in Compile).value,
+          "dotty-library" -> (packageBin in (`dotty-library`, Compile)).value,
+          "dotty-compiler-test" -> (packageBin in Test).value
+        ) map { case (k, v) => (k, v.getAbsolutePath) }
+      },
+
+      // For convenience, change the baseDirectory when running the compiler
+      baseDirectory in (Compile, run) := baseDirectory.value / "..",
+      // .. but not when running partest
+      baseDirectory in (Test, run) := baseDirectory.value,
+
+      repl := Def.inputTaskDyn {
+        val args: Seq[String] = spaceDelimited("<arg>").parsed
+        val dottyLib = packageAll.value("dotty-library")
+        (runMain in Compile).toTask(
+          s" dotty.tools.dotc.repl.Main -classpath $dottyLib " + args.mkString(" ")
+        )
+      }.evaluated,
+
+      // Override run to be able to run compiled classfiles
+      dotr := {
+        val args: Seq[String] = spaceDelimited("<arg>").parsed
+        val java: String = Process("which" :: "java" :: Nil) !!
+        val scalaLib = (dependencyClasspath in Runtime, packageAll)
+          .map { (attList, _) =>
+            attList
+              .map(_.data.getAbsolutePath)
+              .find(_.contains("scala-library"))
+              .toList.mkString(":")
+          }.value
+
+        if (java == "")
+          println("Couldn't find java executable on path, please install java to a default location")
+        else if (scalaLib == "") {
+          println("Couldn't find scala-library on classpath, please run using script in bin dir instead")
+        } else {
+          val dottyLib = packageAll.value("dotty-library")
+          s"""$java -classpath .:$dottyLib:$scalaLib ${args.mkString(" ")}""" !
+        }
+      },
+      run := Def.inputTaskDyn {
+        val dottyLib = packageAll.value("dotty-library")
+        val args: Seq[String] = spaceDelimited("<arg>").parsed
+
+        val fullArgs = args.span(_ != "-classpath") match {
+          case (beforeCp, Nil) => beforeCp ++ ("-classpath" :: dottyLib :: Nil)
+          case (beforeCp, rest) => beforeCp ++ rest
+        }
+
+        (runMain in Compile).toTask(
+          s" dotty.tools.dotc.Main " + fullArgs.mkString(" ")
+        )
+      }.evaluated,
+      dotc := run.evaluated,
+
       // enable verbose exception messages for JUnit
-      testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "--run-listener=test.ContextEscapeDetector"),
+      testOptions in Test += Tests.Argument(
+        TestFrameworks.JUnit, "-a", "-v",
+        "--run-listener=dotty.tools.ContextEscapeDetector"
+      ),
       testOptions in Test += Tests.Cleanup({ () => partestLockFile.delete }),
 
       lockPartestFile := {
@@ -128,9 +259,13 @@ object DottyBuild extends Build {
         // command line arguments get passed to the last task in an aliased
         // sequence (see partest alias below), so this works.
         val args = Def.spaceDelimited("<arg>").parsed
-        val jars = Seq((packageBin in Compile).value.getAbsolutePath) ++
-            getJarPaths(partestDeps.value, ivyPaths.value.ivyHome)
-        val dottyJars  = "-dottyJars " + (jars.length + 1) + " dotty.jar" + " " + jars.mkString(" ")
+        val jars = List(
+          (packageBin in Compile).value.getAbsolutePath,
+          (packageBin in (`dotty-library`, Compile)).value.getAbsolutePath,
+          (packageBin in (`dotty-interfaces`, Compile)).value.getAbsolutePath
+        ) ++ getJarPaths(partestDeps.value, ivyPaths.value.ivyHome)
+        val dottyJars  =
+          s"""-dottyJars ${jars.length + 2} dotty.jar dotty-lib.jar ${jars.mkString(" ")}"""
         // Provide the jars required on the classpath of run tests
         runTask(Test, "dotty.partest.DPConsoleRunner", dottyJars + " " + args.mkString(" "))
       },
@@ -140,35 +275,35 @@ object DottyBuild extends Build {
        * of scalajs-ir built with a different Scala compiler, we add its
        * sources instead of depending on the binaries.
        */
-      ivyConfigurations += config("sourcedeps").hide,
-      libraryDependencies +=
-        "org.scala-js" %% "scalajs-ir" % scalaJSVersion % "sourcedeps",
-      sourceGenerators in Compile += Def.task {
-        val s = streams.value
-        val cacheDir = s.cacheDirectory
-        val trgDir = (sourceManaged in Compile).value / "scalajs-ir-src"
-
-        val report = updateClassifiers.value
-        val scalaJSIRSourcesJar = report.select(
-            configuration = Set("sourcedeps"),
-            module = (_: ModuleID).name.startsWith("scalajs-ir_"),
-            artifact = artifactFilter(`type` = "src")).headOption.getOrElse {
-          sys.error(s"Could not fetch scalajs-ir sources")
-        }
-
-        FileFunction.cached(cacheDir / s"fetchScalaJSIRSource",
-            FilesInfo.lastModified, FilesInfo.exists) { dependencies =>
-          s.log.info(s"Unpacking scalajs-ir sources to $trgDir...")
-          if (trgDir.exists)
-            IO.delete(trgDir)
-          IO.createDirectory(trgDir)
-          IO.unzip(scalaJSIRSourcesJar, trgDir)
-          (trgDir ** "*.scala").get.toSet
-        } (Set(scalaJSIRSourcesJar)).toSeq
-      }.taskValue,
-
-      // Adjust classpath for running dotty
-      mainClass in (Compile, run) := Some("dotty.tools.dotc.Main"),
+      //TODO: disabling until moved to separate project
+      //ivyConfigurations += config("sourcedeps").hide,
+      //libraryDependencies +=
+      //  "org.scala-js" %% "scalajs-ir" % scalaJSVersion % "sourcedeps",
+      //sourceGenerators in Compile += Def.task {
+      //  val s = streams.value
+      //  val cacheDir = s.cacheDirectory
+      //  val trgDir = (sourceManaged in Compile).value / "scalajs-ir-src"
+
+      //  val report = updateClassifiers.value
+      //  val scalaJSIRSourcesJar = report.select(
+      //      configuration = Set("sourcedeps"),
+      //      module = (_: ModuleID).name.startsWith("scalajs-ir_"),
+      //      artifact = artifactFilter(`type` = "src")).headOption.getOrElse {
+      //    sys.error(s"Could not fetch scalajs-ir sources")
+      //  }
+
+      //  FileFunction.cached(cacheDir / s"fetchScalaJSIRSource",
+      //      FilesInfo.lastModified, FilesInfo.exists) { dependencies =>
+      //    s.log.info(s"Unpacking scalajs-ir sources to $trgDir...")
+      //    if (trgDir.exists)
+      //      IO.delete(trgDir)
+      //    IO.createDirectory(trgDir)
+      //    IO.unzip(scalaJSIRSourcesJar, trgDir)
+      //    (trgDir ** "*.scala").get.toSet
+      //  } (Set(scalaJSIRSourcesJar)).toSeq
+      //}.taskValue,
+
+      // Spawn new JVM in run and test
       fork in run := true,
       fork in Test := true,
       parallelExecution in Test := false,
@@ -177,51 +312,71 @@ object DottyBuild extends Build {
       packageOptions += ManifestAttributes(("Git-Hash", VersionUtil.gitHash)),
 
       // http://grokbase.com/t/gg/simple-build-tool/135ke5y90p/sbt-setting-jvm-boot-paramaters-for-scala
-      javaOptions <++= (dependencyClasspath in Runtime, packageBin in Compile) map { (attList, bin) =>
-        // put the Scala {library, reflect} in the classpath
+      // packageAll should always be run before tests
+      javaOptions <++= (dependencyClasspath in Runtime, packageAll) map { (attList, _) =>
+        // put needed dependencies on classpath:
         val path = for {
           file <- attList.map(_.data)
           path = file.getAbsolutePath
+          // FIXME: when we snip the cord, this should go bye-bye
+          if path.contains("scala-library") ||
+            // FIXME: currently needed for tests referencing scalac internals
+            path.contains("scala-reflect") ||
+            // FIXME: currently needed for tests referencing scalac internals
+            path.contains("scala-compile") ||
+            // FIXME: should go away when xml literal parsing is removed
+            path.contains("scala-xml") ||
+            // needed for the xsbti interface
+            path.contains("sbt-interface")
         } yield "-Xbootclasspath/p:" + path
-        // dotty itself needs to be in the bootclasspath
-        val fullpath = ("-Xbootclasspath/p:" + "dotty.jar") :: ("-Xbootclasspath/a:" + bin) :: path.toList
-        // System.err.println("BOOTPATH: " + fullpath)
 
         val travis_build = // propagate if this is a travis build
           if (sys.props.isDefinedAt(JENKINS_BUILD))
             List(s"-D$JENKINS_BUILD=${sys.props(JENKINS_BUILD)}") ::: jenkinsMemLimit
-          else
-            List()
+          else List()
 
         val tuning =
           if (sys.props.isDefinedAt("Oshort"))
             // Optimize for short-running applications, see https://github.com/lampepfl/dotty/issues/222
             List("-XX:+TieredCompilation", "-XX:TieredStopAtLevel=1")
-          else
-            List()
+          else List()
 
-        ("-DpartestParentID=" + pid) :: tuning ::: agentOptions ::: travis_build ::: fullpath
+        ("-DpartestParentID=" + pid) :: tuning ::: agentOptions ::: travis_build ::: path.toList
       }
     ).
-    settings(
-      addCommandAlias("partest",                   ";test:package;package;test:runMain dotc.build;lockPartestFile;test:test;runPartestRunner") ++
-      addCommandAlias("partest-only",              ";test:package;package;test:runMain dotc.build;lockPartestFile;test:test-only dotc.tests;runPartestRunner") ++
-      addCommandAlias("partest-only-no-bootstrap", ";test:package;package;                        lockPartestFile;test:test-only dotc.tests;runPartestRunner") ++
-      addCommandAlias("dottydoc", ";dottydoc/run")
-    ).
     settings(publishing)
 
+  /* Contains unit tests for the scripts */
+  lazy val `dotty-bin-tests` = project.in(file("bin")).
+    settings(sourceStructure).
+    settings(
+      publishArtifact := false,
+      parallelExecution in Test := false,
+      libraryDependencies +=
+        "com.novocode" % "junit-interface" % "0.11" % "test"
+    )
+
+  lazy val `dotty-library` = project.in(file("library")).
+    settings(sourceStructure).
+    settings(
+      libraryDependencies ++= Seq(
+        "org.scala-lang" % "scala-reflect" % scalaVersion.value,
+        "org.scala-lang" % "scala-library" % scalaVersion.value
+      )
+    )
+
   // until sbt/sbt#2402 is fixed (https://github.com/sbt/sbt/issues/2402)
   lazy val cleanSbtBridge = TaskKey[Unit]("cleanSbtBridge", "delete dotty-sbt-bridge cache")
 
-  lazy val `dotty-bridge` = project.in(file("bridge")).
-    dependsOn(dotty).
+  lazy val `dotty-sbt-bridge` = project.in(file("sbt-bridge")).
+    dependsOn(`dotty-compiler`).
+    settings(sourceStructure).
     settings(
       overrideScalaVersionSetting,
 
       cleanSbtBridge := {
         val dottyBridgeVersion = version.value
-        val dottyVersion = (version in dotty).value
+        val dottyVersion = (version in `dotty-compiler`).value
         val classVersion = System.getProperty("java.class.version")
 
         val sbtV = sbtVersion.value
@@ -261,6 +416,7 @@ object DottyBuild extends Build {
     ).
     settings(ScriptedPlugin.scriptedSettings: _*).
     settings(
+      ScriptedPlugin.sbtTestDirectory := baseDirectory.value / "sbt-test",
       ScriptedPlugin.scriptedLaunchOpts := Seq("-Xmx1024m"),
       ScriptedPlugin.scriptedBufferLog := false
       // TODO: Use this instead of manually copying DottyInjectedPlugin.scala
@@ -302,6 +458,7 @@ object DottyInjectedPlugin extends AutoPlugin {
    */
   lazy val sjsSandbox = project.in(file("sandbox/scalajs")).
     enablePlugins(ScalaJSPlugin).
+    settings(sourceStructure).
     settings(
       overrideScalaVersionSetting,
 
@@ -335,11 +492,12 @@ object DottyInjectedPlugin extends AutoPlugin {
     )))
 
   lazy val `dotty-bench` = project.in(file("bench")).
-    dependsOn(dotty % "compile->test").
+    dependsOn(`dotty-compiler` % "compile->test").
+    settings(sourceStructure).
     settings(
       overrideScalaVersionSetting,
 
-      baseDirectory in (Test,run) := (baseDirectory in dotty).value,
+      baseDirectory in (Test,run) := (baseDirectory in `dotty-compiler`).value,
 
       libraryDependencies ++= Seq(
         scalaCompiler % Test,
@@ -474,7 +632,7 @@ object DottyInjectedPlugin extends AutoPlugin {
         def cpToString(cp: Seq[File]) =
           cp.map(_.getAbsolutePath).mkString(java.io.File.pathSeparator)
 
-        val compilerCp = Attributed.data((fullClasspath in (dotty, Compile)).value)
+        val compilerCp = Attributed.data((fullClasspath in (`dotty-compiler`, Compile)).value)
         val cpStr = cpToString(classpath ++ compilerCp)
 
         // List all my dependencies (recompile if any of these changes)