Add flat classpath implementation for zip and jar files

This commit adds an implementation of flat classpath which can handle
both jar and vanilla zip files. In fact there are two versions - for
a class- and a sourcepath. Both extend ZipArchiveFileLookup which
provides common logic.

They use FileZipArchive. @gkossakowski made a comparison of different
ways of handling zips and jars (e.g. using javac's ZipFileIndex). He
stated that general efficiency of FileZipArchive, taking into account
various parameters, is the best.

FileZipArchive is slightly changed. From now it allows to find the
entry for directory in all directory entries without iterating all
entries regardless of a type. Thanks to that we can simply find
a directory for a package - like in the case of DirectoryFileLookup.

There's also added possibility to cache classpath representation of
classpath elements from jar and zip files across compiler instances.
The cache is just a map AbstractFile -> FlatClassPath. It should
reduce the number of created classpath and file instances e.g. in the
case of many ScalaPresentationCompilers in Scala IDE.

To prevent the possibility to avoid a cache, caches are created as
a part of factories responsible for the creation of these types of
the flat classpath.

3 files changed, 154 insertions, 0 deletions

diff --git a/src/compiler/scala/tools/nsc/classpath/FileUtils.scala b/src/compiler/scala/tools/nsc/classpath/FileUtils.scala
index f8f5ac3ac5..cce16957f1 100644
--- a/src/compiler/scala/tools/nsc/classpath/FileUtils.scala
+++ b/src/compiler/scala/tools/nsc/classpath/FileUtils.scala
@@ -13,6 +13,11 @@ import scala.reflect.io.AbstractFile
  */
 object FileUtils {
   implicit class AbstractFileOps(val file: AbstractFile) extends AnyVal {
+    def isPackage: Boolean = file.isDirectory && mayBeValidPackage(file.name)
+
+    def isClass: Boolean = !file.isDirectory && file.hasExtension("class")
+
+    def isScalaOrJavaSource: Boolean = !file.isDirectory && (file.hasExtension("scala") || file.hasExtension("java"))
 
     /**
      * Safe method returning a sequence containing one URL representing this file, when underlying file exists,
diff --git a/src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala b/src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala
new file mode 100644
index 0000000000..5096ca6f70
--- /dev/null
+++ b/src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2014 Contributor. All rights reserved.
+ */
+package scala.tools.nsc.classpath
+
+import java.io.File
+import scala.reflect.io.{ AbstractFile, FileZipArchive }
+import scala.tools.nsc.Settings
+import FileUtils._
+
+/**
+ * A trait providing a cache for classpath entries obtained from zip and jar files.
+ * It's possible to create such a cache assuming that entries in such files won't change (at
+ * least will be the same each time we'll load classpath during the lifetime of JVM process)
+ * - unlike class and source files in directories, which can be modified and recompiled.
+ * It allows us to e.g. reduce significantly memory used by PresentationCompilers in Scala IDE
+ * when there are a lot of projects having a lot of common dependencies.
+ */
+trait ZipAndJarFileLookupFactory {
+
+  private val cache = collection.mutable.Map.empty[AbstractFile, FlatClassPath]
+
+  def create(zipFile: AbstractFile, settings: Settings): FlatClassPath = cache.synchronized {
+    def newClassPathInstance = {
+      if (settings.verbose || settings.Ylogcp)
+        println(s"$zipFile is not yet in the classpath cache")
+      createForZipFile(zipFile)
+    }
+    cache.getOrElseUpdate(zipFile, newClassPathInstance)
+  }
+
+  protected def createForZipFile(zipFile: AbstractFile): FlatClassPath
+}
+
+/**
+ * Manages creation of flat classpath for class files placed in zip and jar files.
+ * It should be the only way of creating them as it provides caching.
+ */
+object ZipAndJarFlatClassPathFactory extends ZipAndJarFileLookupFactory {
+
+  private case class ZipArchiveFlatClassPath(zipFile: File)
+    extends ZipArchiveFileLookup[ClassFileEntryImpl]
+    with NoSourcePaths {
+
+    override def findClassFile(className: String): Option[AbstractFile] = {
+      val (pkg, simpleClassName) = PackageNameUtils.separatePkgAndClassNames(className)
+      classes(pkg).find(_.name == simpleClassName).map(_.file)
+    }
+
+    override private[nsc] def classes(inPackage: String): Seq[ClassFileEntry] = files(inPackage)
+
+    override protected def createFileEntry(file: FileZipArchive#Entry): ClassFileEntryImpl = ClassFileEntryImpl(file)
+    override protected def isRequiredFileType(file: AbstractFile): Boolean = file.isClass
+  }
+
+  override protected def createForZipFile(zipFile: AbstractFile): FlatClassPath =
+    if (zipFile.file == null) {
+      val errorMsg = s"Abstract files which don't have an underlying file are not supported. There was $zipFile"
+      throw new IllegalArgumentException(errorMsg)
+    } else ZipArchiveFlatClassPath(zipFile.file)
+}
+
+/**
+ * Manages creation of flat classpath for source files placed in zip and jar files.
+ * It should be the only way of creating them as it provides caching.
+ */
+object ZipAndJarFlatSourcePathFactory extends ZipAndJarFileLookupFactory {
+
+  private case class ZipArchiveFlatSourcePath(zipFile: File)
+    extends ZipArchiveFileLookup[SourceFileEntryImpl]
+    with NoClassPaths {
+
+    override def asSourcePathString: String = asClassPathString
+
+    override private[nsc] def sources(inPackage: String): Seq[SourceFileEntry] = files(inPackage)
+
+    override protected def createFileEntry(file: FileZipArchive#Entry): SourceFileEntryImpl = SourceFileEntryImpl(file)
+    override protected def isRequiredFileType(file: AbstractFile): Boolean = file.isScalaOrJavaSource
+  }
+
+  override protected def createForZipFile(zipFile: AbstractFile): FlatClassPath = ZipArchiveFlatSourcePath(zipFile.file)
+}
diff --git a/src/compiler/scala/tools/nsc/classpath/ZipArchiveFileLookup.scala b/src/compiler/scala/tools/nsc/classpath/ZipArchiveFileLookup.scala
new file mode 100644
index 0000000000..1d0de57779
--- /dev/null
+++ b/src/compiler/scala/tools/nsc/classpath/ZipArchiveFileLookup.scala
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2014 Contributor. All rights reserved.
+ */
+package scala.tools.nsc.classpath
+
+import java.io.File
+import java.net.URL
+import scala.collection.Seq
+import scala.reflect.io.AbstractFile
+import scala.reflect.io.FileZipArchive
+import FileUtils.AbstractFileOps
+
+/**
+ * A trait allowing to look for classpath entries of given type in zip and jar files.
+ * It provides common logic for classes handling class and source files.
+ * It's aware of things like e.g. META-INF directory which is correctly skipped.
+ */
+trait ZipArchiveFileLookup[FileEntryType <: ClassRepClassPathEntry] extends FlatClassPath {
+  val zipFile: File
+
+  assert(zipFile != null, "Zip file in ZipArchiveFileLookup cannot be null")
+
+  override def asURLs: Seq[URL] = Seq(zipFile.toURI.toURL)
+  override def asClassPathStrings: Seq[String] = Seq(zipFile.getPath)
+
+  private val archive = new FileZipArchive(zipFile)
+
+  override private[nsc] def packages(inPackage: String): Seq[PackageEntry] = {
+    val prefix = PackageNameUtils.packagePrefix(inPackage)
+    for {
+      dirEntry <- findDirEntry(inPackage).toSeq
+      entry <- dirEntry.iterator if entry.isPackage
+    } yield PackageEntryImpl(prefix + entry.name)
+  }
+
+  protected def files(inPackage: String): Seq[FileEntryType] =
+    for {
+      dirEntry <- findDirEntry(inPackage).toSeq
+      entry <- dirEntry.iterator if isRequiredFileType(entry)
+    } yield createFileEntry(entry)
+
+  override private[nsc] def list(inPackage: String): FlatClassPathEntries = {
+    val foundDirEntry = findDirEntry(inPackage)
+
+    foundDirEntry map { dirEntry =>
+      val pkgBuf = collection.mutable.ArrayBuffer.empty[PackageEntry]
+      val fileBuf = collection.mutable.ArrayBuffer.empty[FileEntryType]
+      val prefix = PackageNameUtils.packagePrefix(inPackage)
+
+      for (entry <- dirEntry.iterator) {
+        if (entry.isPackage)
+          pkgBuf += PackageEntryImpl(prefix + entry.name)
+        else if (isRequiredFileType(entry))
+          fileBuf += createFileEntry(entry)
+      }
+      FlatClassPathEntries(pkgBuf, fileBuf)
+    } getOrElse FlatClassPathEntries(Seq.empty, Seq.empty)
+  }
+
+  private def findDirEntry(pkg: String) = {
+    val dirName = s"${FileUtils.dirPath(pkg)}/"
+    archive.allDirs.get(dirName)
+  }
+
+  protected def createFileEntry(file: FileZipArchive#Entry): FileEntryType
+  protected def isRequiredFileType(file: AbstractFile): Boolean
+}