Scala signature is generated as an annotation (...

Scala signature is generated as an annotation (that is accessible
through Java reflection).

  - compiler generates all pickled Scala signatures as annotations to class files.
  - compiler can read class files with signature as annotations or old-style signatures as attributes.
  - Scalap has also been updated to new signatures (contributed by Ilya Sergey: thanks a lot).
  - FJBG updated to allow entering constant pool strings as byte arrays.
  - ByteCodecs decode method returns the length of the decoded array.
Review by ilyas. Already mostly reviewed by odersky.

5 files changed, 127 insertions, 29 deletions

diff --git a/src/compiler/scala/tools/nsc/backend/jvm/GenJVM.scala b/src/compiler/scala/tools/nsc/backend/jvm/GenJVM.scala
index eccce2922e..147b25a70c 100644
--- a/src/compiler/scala/tools/nsc/backend/jvm/GenJVM.scala
+++ b/src/compiler/scala/tools/nsc/backend/jvm/GenJVM.scala
@@ -8,7 +8,6 @@
 package scala.tools.nsc
 package backend.jvm
 
-import java.io.{ DataOutputStream, File, OutputStream }
 import java.nio.ByteBuffer
 
 import scala.collection.immutable.{Set, ListSet}
@@ -18,6 +17,8 @@ import scala.tools.nsc.symtab._
 import scala.tools.nsc.symtab.classfile.ClassfileConstants._
 
 import ch.epfl.lamp.fjbg._
+import java.io.{ByteArrayOutputStream, DataOutputStream, File, OutputStream}
+import reflect.generic.{PickleFormat, PickleBuffer}
 
 /** This class ...
  *
@@ -104,6 +105,16 @@ abstract class GenJVM extends SubComponent {
     lazy val RemoteInterface = definitions.getClass("java.rmi.Remote")
     lazy val RemoteException = definitions.getClass("java.rmi.RemoteException").tpe
 
+
+    val versionPickle = {
+      val vp = new PickleBuffer(new Array[Byte](16), -1, 0)
+      assert(vp.writeIndex == 0)
+      vp.writeNat(PickleFormat.MajorVersion)
+      vp.writeNat(PickleFormat.MinorVersion)
+      vp.writeNat(0)
+      vp
+    }
+
     var clasz: IClass = _
     var method: IMethod = _
     var jclass: JClass = _
@@ -125,25 +136,6 @@ abstract class GenJVM extends SubComponent {
      * @param sym    The corresponding symbol, used for looking up pickled information
      */
     def emitClass(jclass: JClass, sym: Symbol) {
-      def addScalaAttr(sym: Symbol): Unit = currentRun.symData.get(sym) match {
-        case Some(pickle) =>
-          val scalaAttr = fjbgContext.JOtherAttribute(jclass,
-                                                  jclass,
-                                                  nme.ScalaSignatureATTR.toString,
-                                                  pickle.bytes,
-                                                  pickle.writeIndex)
-          pickledBytes = pickledBytes + pickle.writeIndex
-          jclass.addAttribute(scalaAttr)
-          currentRun.symData -= sym
-          currentRun.symData -= sym.companionSymbol
-          //System.out.println("Generated ScalaSig Attr for " + sym)//debug
-        case _ =>
-          val markerAttr = getMarkerAttr(jclass)
-          jclass.addAttribute(markerAttr)
-          log("Could not find pickle information for " + sym)
-      }
-      if (!(jclass.getName().endsWith("$") && sym.isModuleClass))
-        addScalaAttr(if (isTopLevelModule(sym)) sym.sourceModule else sym);
       addInnerClasses(jclass)
 
       val outfile = getFile(sym, jclass, ".class")
@@ -153,6 +145,43 @@ abstract class GenJVM extends SubComponent {
       informProgress("wrote " + outfile)
     }
 
+    /** Returns the ScalaSignature annotation if it must be added to this class, none otherwise; furthermore, it adds to
+      * jclass the Scala marker attribute (marking that a scala signature annotation is present). The annotation that is
+      * returned by this method must be added to the class' annotations list when generating them.
+      * @param jclass The class file that is being readied.
+      * @param sym    The symbol for which the signature has been entered in the symData map. This is different than the
+      *               symbol that is being generated in the case of a mirror class.
+      * @return       An option that is:
+      *                - defined and contains an annotation info of the ScalaSignature type, instantiated with the
+      *                  pickle signature for sym;
+      *                - undefined if the jclass/sym couple must not contain a signature or if there was an error (see
+      *                  log output in debug mode). */
+    def scalaSignatureAddingMarker(jclass: JClass, sym: Symbol): Option[AnnotationInfo] =
+      if (jclass.getName().endsWith("$") && sym.isModuleClass) {
+        None
+      }
+      else {
+        val sd = currentRun.symData
+        currentRun.symData.get(sym) match {
+          case Some(pickle) =>
+            val scalaAttr =
+              fjbgContext.JOtherAttribute(jclass, jclass, nme.ScalaSignatureATTR.toString,
+                                          versionPickle.bytes, versionPickle.writeIndex)
+            jclass.addAttribute(scalaAttr)
+            val scalaAnnot =
+              AnnotationInfo(definitions.ScalaSignatureAnnotation.tpe, Nil, List(
+                (nme.bytes, ScalaSigBytes(pickle.bytes.take(pickle.writeIndex)))
+              ))
+            pickledBytes = pickledBytes + pickle.writeIndex
+            currentRun.symData -= sym
+            currentRun.symData -= sym.companionSymbol
+            Some(scalaAnnot)
+          case _ =>
+            log("Could not find annotation pickle information for " + sym)
+            None
+        }
+      }
+
     private def getMarkerAttr(jclass: JClass): JOtherAttribute =
       fjbgContext.JOtherAttribute(jclass, jclass, nme.ScalaATTR.toString, new Array[Byte](0), 0)
 
@@ -237,8 +266,9 @@ abstract class GenJVM extends SubComponent {
       clasz.fields foreach genField
       clasz.methods foreach genMethod
 
+      val ssa = scalaSignatureAddingMarker(jclass, c.symbol)
       addGenericSignature(jclass, c.symbol, c.symbol.owner)
-      addAnnotations(jclass, c.symbol.annotations)
+      addAnnotations(jclass, c.symbol.annotations ++ ssa)
       emitClass(jclass, c.symbol)
 
       if (c.symbol hasAnnotation BeanInfoAttr)
@@ -395,6 +425,10 @@ abstract class GenJVM extends SubComponent {
               buf.putShort(cpool.addUtf8(const.symbolValue.name.toString).toShort)
           }
 
+        case ScalaSigBytes(bytes) =>
+          buf.put('s'.toByte)
+          buf.putShort(cpool.addUtf8(reflect.generic.ByteCodecs.encode(bytes)).toShort)
+
         case ArrayAnnotArg(args) =>
           buf.put('['.toByte)
           buf.putShort(args.length.toShort)
@@ -883,6 +917,8 @@ abstract class GenJVM extends SubComponent {
                                            JClass.NO_INTERFACES,
                                            sourceFile)
       addForwarders(mirrorClass, clasz)
+      val ssa = scalaSignatureAddingMarker(mirrorClass, clasz.companionSymbol)
+      addAnnotations(mirrorClass, clasz.annotations ++ ssa)
       emitClass(mirrorClass, clasz)
     }
 
diff --git a/src/compiler/scala/tools/nsc/symtab/AnnotationInfos.scala b/src/compiler/scala/tools/nsc/symtab/AnnotationInfos.scala
index f0265c5e5d..f32c232316 100644
--- a/src/compiler/scala/tools/nsc/symtab/AnnotationInfos.scala
+++ b/src/compiler/scala/tools/nsc/symtab/AnnotationInfos.scala
@@ -42,6 +42,15 @@ trait AnnotationInfos extends reflect.generic.AnnotationInfos { self: SymbolTabl
 
   object ArrayAnnotArg extends ArrayAnnotArgExtractor
 
+  /** A specific annotation argument that encodes an array of bytes as an array of `Long`. The type of the argument
+    * declared in the annotation must be `String`. This specialised class is used to encode scala signatures for
+    * reasons of efficiency, both in term of class-file size and in term of compiler performance. */
+  case class ScalaSigBytes(bytes: Array[Byte])
+  extends ClassfileAnnotArg {
+    override def toString = (bytes map { byte => (byte & 0xff).toHexString }).mkString("[ ", " ", " ]")
+  }
+  object ScalaSigBytes extends ScalaSigBytesExtractor
+
   /** Represents a nested classfile annotation */
   case class NestedAnnotArg(annInfo: AnnotationInfo)
   extends ClassfileAnnotArg {
diff --git a/src/compiler/scala/tools/nsc/symtab/Definitions.scala b/src/compiler/scala/tools/nsc/symtab/Definitions.scala
index d3b1d4dabe..2d4d01623d 100644
--- a/src/compiler/scala/tools/nsc/symtab/Definitions.scala
+++ b/src/compiler/scala/tools/nsc/symtab/Definitions.scala
@@ -238,6 +238,8 @@ trait Definitions extends reflect.generic.StandardDefinitions {
     lazy val CodeModule           = getModule(sn.Code)
       def Code_lift = getMember(CodeModule, nme.lift_)
 
+    lazy val ScalaSignatureAnnotation = getClass("scala.reflect.ScalaSignature")
+
     // invoke dynamic support
     lazy val LinkageModule = getModule("java.dyn.Linkage")
       lazy val Linkage_invalidateCallerClass = getMember(LinkageModule, "invalidateCallerClass")
diff --git a/src/compiler/scala/tools/nsc/symtab/StdNames.scala b/src/compiler/scala/tools/nsc/symtab/StdNames.scala
index 5ce0025de5..8fbdf680e6 100644
--- a/src/compiler/scala/tools/nsc/symtab/StdNames.scala
+++ b/src/compiler/scala/tools/nsc/symtab/StdNames.scala
@@ -263,6 +263,7 @@ trait StdNames extends reflect.generic.StdNames { self: SymbolTable =>
     val assume_ = newTermName("assume")
     val asInstanceOf_ = newTermName("asInstanceOf")
     val box = newTermName("box")
+    val bytes = newTermName("bytes")
     val canEqual_ = newTermName("canEqual")
     val checkInitialized = newTermName("checkInitialized")
     val classOf = newTermName("classOf")
diff --git a/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala b/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala
index e8b2b84653..aec2d9dda8 100644
--- a/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala
+++ b/src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala
@@ -15,6 +15,7 @@ import scala.collection.immutable.{Map, ListMap}
 import scala.collection.mutable.{ListBuffer, ArrayBuffer}
 import scala.tools.nsc.io.AbstractFile
 import scala.annotation.switch
+import reflect.generic.PickleBuffer
 
 /** This abstract class implements a class file parser.
  *
@@ -35,6 +36,7 @@ abstract class ClassfileParser {
   protected var staticDefs: Scope = _       // the scope of all static definitions
   protected var pool: ConstantPool = _      // the classfile's constant pool
   protected var isScala: Boolean = _        // does class file describe a scala class?
+  protected var isScalaAnnot: Boolean = _   // does class file describe a scala class with its pickled info in an annotation?
   protected var isScalaRaw: Boolean = _     // this class file is a scala class with no pickled info
   protected var hasMeta: Boolean = _        // does class file contain jaco meta attribute?s
   protected var busy: Option[Symbol] = None // lock to detect recursive reads
@@ -353,6 +355,22 @@ abstract class ClassfileParser {
       }
     }
 
+    def getBytes(index: Int): Array[Byte] = {
+      if (index <= 0 || len <= index) errorBadIndex(index)
+      var value = values(index).asInstanceOf[Array[Byte]]
+      if (value eq null) {
+        val start = starts(index)
+        if (in.buf(start).toInt != CONSTANT_UTF8) errorBadTag(start)
+        val len = in.getChar(start + 1)
+        val bytes = new Array[Byte](len)
+        Array.copy(in.buf, start + 3, bytes, 0, len)
+        val decodedLength = reflect.generic.ByteCodecs.decode(bytes)
+        value = bytes.take(decodedLength)
+        values(index) = value
+      }
+      value
+    }
+
     /** Throws an exception signaling a bad constant index. */
     private def errorBadIndex(index: Int) =
       throw new RuntimeException("bad constant pool index: " + index + " at pos: " + in.bp)
@@ -786,9 +804,12 @@ abstract class ClassfileParser {
           if (c1 ne null) sym.setInfo(ConstantType(c1))
           else println("failure to convert " + c + " to " + symtype); //debug
         case nme.ScalaSignatureATTR =>
-          unpickler.unpickle(in.buf, in.bp, clazz, staticModule, in.file.toString())
+          if (!isScalaAnnot) {
+            if (settings.debug.value)
+              global.inform("warning: symbol " + sym.fullName + " has pickled signature in attribute")
+            unpickler.unpickle(in.buf, in.bp, clazz, staticModule, in.file.toString())
+          }
           in.skip(attrLen)
-          this.isScala = true
         case nme.ScalaATTR =>
           isScalaRaw = true
         case nme.JacoMetaATTR =>
@@ -801,9 +822,19 @@ abstract class ClassfileParser {
           in.skip(attrLen)
         // Java annotatinos on classes / methods / fields with RetentionPolicy.RUNTIME
         case nme.RuntimeAnnotationATTR =>
-          if (!isScala) {
-            // no need to read annotations if isScala, ClassfileAnnotations are pickled
+          if (isScalaAnnot || !isScala) {
             parseAnnotations(attrLen)
+            if (isScalaAnnot)
+              (sym.rawAnnotations find { annot =>
+                annot.asInstanceOf[AnnotationInfo].atp == definitions.ScalaSignatureAnnotation.tpe
+              }) match {
+                case Some(san: AnnotationInfo) =>
+                  val bytes =
+                    san.assocs.find({ _._1 == nme.bytes }).get._2.asInstanceOf[ScalaSigBytes].bytes
+                  unpickler.unpickle(bytes, 0, clazz, staticModule, in.file.toString())
+                case None =>
+                  throw new RuntimeException("Scala class file does not contain Scala annotation")
+              }
             if (settings.debug.value)
               global.inform("" + sym + "; annotations = " + sym.annotations)
           } else
@@ -862,6 +893,12 @@ abstract class ClassfileParser {
       }
     }
 
+    def parseScalaSigBytes: Option[ScalaSigBytes] = {
+      val tag = in.nextByte.toChar
+      assert(tag == STRING_TAG)
+      Some(ScalaSigBytes(pool.getBytes(in.nextChar)))
+    }
+
     /** Parse and return a single annotation.  If it is malformed,
      *  return None.
      */
@@ -872,10 +909,19 @@ abstract class ClassfileParser {
       var hasError = false
       for (i <- 0 until nargs) {
         val name = pool.getName(in.nextChar)
-        parseAnnotArg match {
-          case Some(c) => nvpairs += ((name, c))
-          case None => hasError = true
-        }
+        // The "bytes: String" argument of the ScalaSignature attribute is parsed specially so that it is
+        // available as an array of bytes (the pickled Scala signature) instead of as a string. The pickled signature
+        // is encoded as a string because of limitations in the Java class file format.
+        if ((attrType == definitions.ScalaSignatureAnnotation.tpe) && (name == nme.bytes))
+          parseScalaSigBytes match {
+            case Some(c) => nvpairs += ((name, c))
+            case None => hasError = true
+          }
+        else
+          parseAnnotArg match {
+            case Some(c) => nvpairs += ((name, c))
+            case None => hasError = true
+          }
       }
       if (hasError) None
       else Some(AnnotationInfo(attrType, List(), nvpairs.toList))
@@ -985,6 +1031,10 @@ abstract class ClassfileParser {
           in.skip(attrLen)
         case nme.ScalaSignatureATTR =>
           isScala = true
+          val pbuf = new PickleBuffer(in.buf, in.bp, in.bp + attrLen)
+          pbuf.readNat; pbuf.readNat;
+          if (pbuf.readNat == 0) // a scala signature attribute with no entries means that the actual scala signature
+            isScalaAnnot = true    // is in a ScalaSignature annotation.
           in.skip(attrLen)
         case nme.ScalaATTR =>
           isScalaRaw = true