SI-6502 Reenables loading jars into the running REPL (regression in 2.10)

Fixes SI-6502, reenables loading jars into the running REPL
(regression in 2.10). This PR allows adding a jar to the compile
and runtime classpaths without resetting the REPL state (crucial
for Spark SPARK-3257).

This follows the lead taken by @som-snytt in PR #3986, which
differentiates two jar-loading behaviors (muddled by cp):

  - adding jars and replaying REPL expressions (using replay)
  - adding jars without resetting the REPL (deprecated cp,
    introduced require) This PR implements require (left
    unimplemented in #3986)

This PR is a simplification of a similar approach taken by
@gkossakowski in #3884. In this attempt, we check first to make
sure that a jar is only added if it only contains new
classes/traits/objects, otherwise we emit an error. This differs
from the old invalidation approach which also tracked deleted
classpath entries.

2 files changed, 119 insertions, 8 deletions

diff --git a/src/repl/scala/tools/nsc/interpreter/ILoop.scala b/src/repl/scala/tools/nsc/interpreter/ILoop.scala
index 6e18682494..9cbb6ae574 100644
--- a/src/repl/scala/tools/nsc/interpreter/ILoop.scala
+++ b/src/repl/scala/tools/nsc/interpreter/ILoop.scala
@@ -221,7 +221,7 @@ class ILoop(in0: Option[BufferedReader], protected val out: JPrintWriter)
     nullary("power", "enable power user mode", powerCmd),
     nullary("quit", "exit the interpreter", () => Result(keepRunning = false, None)),
     cmd("replay", "[options]", "reset the repl and replay all previous commands", replayCommand),
-    //cmd("require", "<path>", "add a jar or directory to the classpath", require),  // TODO
+    cmd("require", "<path>", "add a jar or directory to the classpath", require),
     cmd("reset", "[options]", "reset the repl to its initial state, forgetting all session entries", resetCommand),
     cmd("save", "<path>", "save replayable session to a file", saveCommand),
     shCommand,
@@ -612,13 +612,73 @@ class ILoop(in0: Option[BufferedReader], protected val out: JPrintWriter)
     val f = File(arg).normalize
     if (f.exists) {
       addedClasspath = ClassPath.join(addedClasspath, f.path)
-      val totalClasspath = ClassPath.join(settings.classpath.value, addedClasspath)
-      echo("Added '%s'.  Your new classpath is:\n\"%s\"".format(f.path, totalClasspath))
-      replay()
+      intp.addUrlsToClassPath(f.toURI.toURL)
+      echo("Added '%s'.  Your new classpath is:\n\"%s\"".format(f.path, intp.global.classPath.asClasspathString))
     }
     else echo("The path '" + f + "' doesn't seem to exist.")
   }
 
+  /** Adds jar file to the current classpath. Jar will only be added if it
+   *  does not contain classes that already exist on the current classpath.
+   *
+   *  Importantly, `require` adds jars to the classpath ''without'' resetting
+   *  the state of the interpreter. This is in contrast to `replay` which can
+   *  be used to add jars to the classpath and which creates a new instance of
+   *  the interpreter and replays all interpreter expressions.
+   */
+  def require(arg: String): Unit = {
+    class InfoClassLoader extends java.lang.ClassLoader {
+      def classOf(arr: Array[Byte]): Class[_] =
+        super.defineClass(null, arr, 0, arr.length)
+    }
+
+    def readFully(is: InputStream): Array[Byte] = {
+      val dis = new java.io.DataInputStream(is)
+      val len = dis.available()
+      val arr = Array.ofDim[Byte](len)
+      dis.readFully(arr)
+      dis.close()
+      arr
+    }
+
+    val f = File(arg).normalize
+
+    if (f.isDirectory) {
+      echo("Adding directories to the classpath is not supported. Add a jar instead.")
+      return
+    }
+    val jarFile = new java.util.jar.JarFile(arg)
+    val entries = jarFile.entries
+    val cloader = new InfoClassLoader
+    var exists = false
+
+    while (entries.hasMoreElements()) {
+      val entry = entries.nextElement()
+      // skip directories and manifests
+      if (!entry.isDirectory() && !entry.getName().endsWith("MF")) {
+        // for each entry get InputStream
+        val is = jarFile.getInputStream(entry)
+        // read InputStream into Array[Byte]
+        val arr = readFully(is)
+        val clazz = cloader.classOf(arr)
+        try {
+          Class.forName(clazz.getName(), false, intp.classLoader)
+          exists = true
+        } catch {
+          case _: ClassNotFoundException => /* do nothing */
+        }
+      }
+    }
+
+    if (f.exists && !exists) {
+      addedClasspath = ClassPath.join(addedClasspath, f.path)
+      intp.addUrlsToClassPath(f.toURI.toURL)
+      echo("Added '%s'.  Your new classpath is:\n\"%s\"".format(f.path, intp.global.classPath.asClasspathString))
+    } else if (exists) {
+      echo("The path '" + f + "' cannot be loaded, because existing classpath entries conflict.")
+    } else echo("The path '" + f + "' doesn't seem to exist.")
+  }
+
   def powerCmd(): Result = {
     if (isReplPower) "Already in power mode."
     else enablePowerMode(isDuringInit = false)
diff --git a/src/repl/scala/tools/nsc/interpreter/IMain.scala b/src/repl/scala/tools/nsc/interpreter/IMain.scala
index 3f4922a602..4f035bae7a 100644
--- a/src/repl/scala/tools/nsc/interpreter/IMain.scala
+++ b/src/repl/scala/tools/nsc/interpreter/IMain.scala
@@ -18,9 +18,13 @@ import scala.reflect.internal.util.{ BatchSourceFile, SourceFile }
 import scala.tools.util.PathResolver
 import scala.tools.nsc.io.AbstractFile
 import scala.tools.nsc.typechecker.{ TypeStrings, StructuredTypeStrings }
-import scala.tools.nsc.util.{ ScalaClassLoader, stringFromReader, stringFromWriter, StackTraceOps }
+import scala.tools.nsc.util.{ ScalaClassLoader, stringFromReader, stringFromWriter, StackTraceOps, ClassPath, MergedClassPath }
+import ScalaClassLoader.URLClassLoader
 import scala.tools.nsc.util.Exceptional.unwrap
+import scala.tools.nsc.backend.JavaPlatform
 import javax.script.{AbstractScriptEngine, Bindings, ScriptContext, ScriptEngine, ScriptEngineFactory, ScriptException, CompiledScript, Compilable}
+import java.net.URL
+import java.io.File
 
 /** An interpreter for Scala code.
  *
@@ -82,6 +86,9 @@ class IMain(@BeanProperty val factory: ScriptEngineFactory, initialSettings: Set
   private var _classLoader: util.AbstractFileClassLoader = null                              // active classloader
   private val _compiler: ReplGlobal                 = newCompiler(settings, reporter)   // our private compiler
 
+  private trait ExposeAddUrl extends URLClassLoader { def addNewUrl(url: URL) = this.addURL(url) }
+  private var _runtimeClassLoader: URLClassLoader with ExposeAddUrl = null              // wrapper exposing addURL
+
   def compilerClasspath: Seq[java.net.URL] = (
     if (isInitializeComplete) global.classPath.asURLs
     else new PathResolver(settings).result.asURLs  // the compiler's classpath
@@ -237,6 +244,50 @@ class IMain(@BeanProperty val factory: ScriptEngineFactory, initialSettings: Set
     new Global(settings, reporter) with ReplGlobal { override def toString: String = "<global>" }
   }
 
+  /**
+   * Adds all specified jars to the compile and runtime classpaths.
+   *
+   * @note  Currently only supports jars, not directories.
+   * @param urls The list of items to add to the compile and runtime classpaths.
+   */
+  def addUrlsToClassPath(urls: URL*): Unit = {
+    new Run //  force some initialization
+    urls.foreach(_runtimeClassLoader.addNewUrl) // Add jars to runtime classloader
+    extendCompilerClassPath(urls: _*)           // Add jars to compile-time classpath
+  }
+
+  /** Extend classpath of global.platform and force `global` to rescan updated packages. */
+  protected def extendCompilerClassPath(urls: URL*): Unit = {
+    val newClassPath = mergeUrlsIntoClassPath(global.platform, urls: _*)
+    global.platform.currentClassPath = Some(newClassPath)
+    // Reload all specified jars into the current compiler instance (global)
+    global.invalidateClassPathEntries(urls.map(_.getPath): _*)
+  }
+
+  /** Merge classpath of `platform` and `urls` into merged classpath */
+  protected def mergeUrlsIntoClassPath(platform: JavaPlatform, urls: URL*): MergedClassPath[AbstractFile] = {
+    // Collect our new jars/directories and add them to the existing set of classpaths
+    val prevEntries = platform.classPath match {
+      case mcp: MergedClassPath[AbstractFile] => mcp.entries
+      case cp:  ClassPath[AbstractFile]       => List(cp)
+    }
+    val allEntries = (prevEntries ++
+      urls.map(url => platform.classPath.context.newClassPath(
+          if (url.getProtocol == "file") {
+            val f = new File(url.getPath)
+            if (f.isDirectory) io.AbstractFile.getDirectory(f)
+            else io.AbstractFile.getFile(f)
+          } else {
+            io.AbstractFile.getURL(url)
+          }
+        )
+      )
+    ).distinct
+
+    // Combine all of our classpaths (old and new) into one merged classpath
+    new MergedClassPath(allEntries, platform.classPath.context)
+  }
+
   /** Parent classloader.  Overridable. */
   protected def parentClassLoader: ClassLoader =
     settings.explicitParentLoader.getOrElse( this.getClass.getClassLoader() )
@@ -329,9 +380,9 @@ class IMain(@BeanProperty val factory: ScriptEngineFactory, initialSettings: Set
     }
   }
   private def makeClassLoader(): util.AbstractFileClassLoader =
-    new TranslatingClassLoader(parentClassLoader match {
-      case null   => ScalaClassLoader fromURLs compilerClasspath
-      case p      => new ScalaClassLoader.URLClassLoader(compilerClasspath, p)
+    new TranslatingClassLoader({
+      _runtimeClassLoader = new URLClassLoader(compilerClasspath, parentClassLoader) with ExposeAddUrl
+      _runtimeClassLoader
     })
 
   // Set the current Java "context" class loader to this interpreter's class loader