Merge branch 'develop'

4 files changed, 147 insertions, 52 deletions

diff --git a/src/compiler/scala/reflect/internal/Importers.scala b/src/compiler/scala/reflect/internal/Importers.scala
index c232e3b7c1..1ae4f755ed 100644
--- a/src/compiler/scala/reflect/internal/Importers.scala
+++ b/src/compiler/scala/reflect/internal/Importers.scala
@@ -286,7 +286,7 @@ trait Importers { self: SymbolTable =>
       new Modifiers(mods.flags, importName(mods.privateWithin), mods.annotations map importTree)
 
     def importImportSelector(sel: from.ImportSelector): ImportSelector =
-      new ImportSelector(importName(sel.name), sel.namePos, importName(sel.rename), sel.renamePos)
+      new ImportSelector(importName(sel.name), sel.namePos, if (sel.rename != null) importName(sel.rename) else null, sel.renamePos)
 
     def importTree(tree: from.Tree): Tree = {
       val mytree = tree match {
diff --git a/src/compiler/scala/reflect/runtime/ToolBoxes.scala b/src/compiler/scala/reflect/runtime/ToolBoxes.scala
index 880c68eaa0..f52662ce6f 100644
--- a/src/compiler/scala/reflect/runtime/ToolBoxes.scala
+++ b/src/compiler/scala/reflect/runtime/ToolBoxes.scala
@@ -64,7 +64,7 @@ trait ToolBoxes extends { self: Universe =>
         obj setInfo obj.moduleClass.tpe
         val meth = obj.moduleClass.newMethod(newTermName(wrapperMethodName))
         def makeParam(fv: Symbol) = meth.newValueParameter(fv.name.toTermName) setInfo fv.tpe
-        meth setInfo MethodType(fvs map makeParam, expr.tpe)
+        meth setInfo MethodType(fvs map makeParam, AnyClass.tpe)
         minfo.decls enter meth
         trace("wrapping ")(defOwner(expr) -> meth)
         val methdef = DefDef(meth, expr changeOwner (defOwner(expr) -> meth))
@@ -94,6 +94,20 @@ trait ToolBoxes extends { self: Universe =>
       }
 
       def compileExpr(expr: Tree, fvs: List[Symbol]): String = {
+        // Previously toolboxes used to typecheck their inputs before compiling.
+        // Actually, the initial demo by Martin first typechecked the reified tree,
+        // then ran it, which typechecked it again, and only then launched the
+        // reflective compiler.
+        //
+        // However, as observed in https://issues.scala-lang.org/browse/SI-5464
+        // current implementation typechecking is not always idempotent.
+        // That's why we cannot allow inputs of toolboxes to be typechecked,
+        // at least not until the aforementioned issue is closed.
+        val typed = expr filter (t => t.tpe != null && t.tpe != NoType && !t.isInstanceOf[TypeTree])
+        if (!typed.isEmpty) {
+          throw new Error("cannot compile trees that are already typed")
+        }
+
         val mdef = wrapInObject(expr, fvs)
         val pdef = wrapInPackage(mdef)
         val unit = wrapInCompilationUnit(pdef)
@@ -106,7 +120,6 @@ trait ToolBoxes extends { self: Universe =>
         jclazz.getDeclaredMethods.find(_.getName == name).get
 
       def runExpr(expr: Tree): Any = {
-        val etpe = expr.tpe
         val fvs = (expr filter isFree map (_.symbol)).distinct
 
         reporter.reset()
@@ -181,19 +194,13 @@ trait ToolBoxes extends { self: Universe =>
 
     lazy val classLoader = new AbstractFileClassLoader(virtualDirectory, defaultReflectiveClassLoader)
 
-    private def importAndTypeCheck(tree: rm.Tree, expectedType: rm.Type): compiler.Tree = {
+    def typeCheck(tree: rm.Tree, expectedType: rm.Type): rm.Tree = {
+      if (compiler.settings.verbose.value) println("typing "+tree+", pt = "+expectedType)
       val ctree: compiler.Tree = importer.importTree(tree.asInstanceOf[Tree])
       val pt: compiler.Type = importer.importType(expectedType.asInstanceOf[Type])
-//      val typer = compiler.typer.atOwner(ctree, if (owner.isModule) cowner.moduleClass else cowner)
       val ttree: compiler.Tree = compiler.typedTopLevelExpr(ctree, pt)
-      ttree
-    }
-
-    def typeCheck(tree: rm.Tree, expectedType: rm.Type): rm.Tree = {
-      if (compiler.settings.verbose.value) println("typing "+tree+", pt = "+expectedType)
-      val ttree = importAndTypeCheck(tree, expectedType)
-      val ettree = exporter.importTree(ttree).asInstanceOf[rm.Tree]
-      ettree
+      val rmttree = exporter.importTree(ttree).asInstanceOf[rm.Tree]
+      rmttree
     }
 
     def typeCheck(tree: rm.Tree): rm.Tree =
@@ -202,11 +209,10 @@ trait ToolBoxes extends { self: Universe =>
     def showAttributed(tree: rm.Tree, printTypes: Boolean = true, printIds: Boolean = true, printKinds: Boolean = false): String =
       compiler.showAttributed(importer.importTree(tree.asInstanceOf[Tree]), printTypes, printIds, printKinds)
 
-    def runExpr(tree: rm.Tree, expectedType: rm.Type): Any = {
-      val ttree = importAndTypeCheck(tree, expectedType)
-      compiler.runExpr(ttree)
+    def runExpr(tree: rm.Tree): Any = {
+      if (compiler.settings.verbose.value) println("running "+tree)
+      val ctree: compiler.Tree = importer.importTree(tree.asInstanceOf[Tree])
+      compiler.runExpr(ctree)
     }
-
-    def runExpr(tree: rm.Tree): Any = runExpr(tree, WildcardType.asInstanceOf[rm.Type])
   }
 }
diff --git a/src/compiler/scala/tools/nsc/ast/Reifiers.scala b/src/compiler/scala/tools/nsc/ast/Reifiers.scala
index 91d5d2bf4a..21e075950f 100644
--- a/src/compiler/scala/tools/nsc/ast/Reifiers.scala
+++ b/src/compiler/scala/tools/nsc/ast/Reifiers.scala
@@ -8,6 +8,7 @@ package ast
 
 import symtab._
 import Flags._
+import scala.reflect.api.Modifier._
 import scala.collection.{ mutable, immutable }
 import scala.collection.mutable.ListBuffer
 import scala.tools.nsc.util.FreshNameCreator
@@ -289,10 +290,102 @@ trait Reifiers { self: Global =>
       var reifySymbols = false
       var reifyTypes = false
 
+      /** Preprocess a tree before reification */
+      private def trimTree(tree: Tree): Tree = {
+        def trimSyntheticCaseClassMembers(deff: Tree, stats: List[Tree]) = {
+          var stats1 = stats filterNot (stat => stat.isDef && {
+            if (stat.symbol.isCaseAccessorMethod && reifyDebug) println("discarding case accessor method: " + stat)
+            stat.symbol.isCaseAccessorMethod
+          })
+          stats1 = stats1 filterNot (memberDef => memberDef.isDef && {
+            val isSynthetic = memberDef.symbol.isSynthetic
+            // @xeno.by: this doesn't work for local classes, e.g. for ones that are top-level to a quasiquote (see comments to companionClass)
+            // that's why I replace the check with an assumption that all synthetic members are, in fact, generated of case classes
+//            val isCaseMember = deff.symbol.isCaseClass || deff.symbol.companionClass.isCaseClass
+            val isCaseMember = true
+            if (isSynthetic && isCaseMember && reifyDebug) println("discarding case class synthetic def: " + memberDef)
+            isSynthetic && isCaseMember
+          })
+          stats1 = stats1 map {
+            case valdef @ ValDef(mods, name, tpt, rhs) if valdef.symbol.isCaseAccessor =>
+              if (reifyDebug) println("resetting visibility of case accessor field: " + valdef)
+              val Modifiers(flags, privateWithin, annotations) = mods
+              val flags1 = flags & ~Flags.LOCAL & ~Flags.PRIVATE
+              val mods1 = Modifiers(flags1, privateWithin, annotations)
+              ValDef(mods1, name, tpt, rhs).copyAttrs(valdef)
+            case stat =>
+              stat
+          }
+          stats1
+        }
+
+        def trimSyntheticCaseClassCompanions(stats: List[Tree]) =
+          stats diff (stats collect { case moddef: ModuleDef => moddef } filter (moddef => {
+            val isSynthetic = moddef.symbol.isSynthetic
+            // @xeno.by: this doesn't work for local classes, e.g. for ones that are top-level to a quasiquote (see comments to companionClass)
+            // that's why I replace the check with an assumption that all synthetic modules are, in fact, companions of case classes
+//            val isCaseCompanion = moddef.symbol.companionClass.isCaseClass
+            val isCaseCompanion = true
+            // @xeno.by: we also have to do this ugly hack for the very same reason described above
+            // normally this sort of stuff is performed in reifyTree, which binds related symbols, however, local companions will be out of its reach
+            if (reifyDebug) println("boundSym: "+  moddef.symbol)
+            boundSyms += moddef.symbol
+            if (isSynthetic && isCaseCompanion && reifyDebug) println("discarding synthetic case class companion: " + moddef)
+            isSynthetic && isCaseCompanion
+          }))
+
+        tree match {
+          case tree if tree.isErroneous =>
+            tree
+          case ta @ TypeApply(hk, ts) =>
+            def isErased(tt: TypeTree) = tt.tpe != null && definedInLiftedCode(tt.tpe) && tt.original == null
+            val discard = ts collect { case tt: TypeTree => tt } exists isErased
+            if (reifyDebug && discard) println("discarding TypeApply: " + tree)
+            if (discard) hk else ta
+          case classDef @ ClassDef(mods, name, params, impl) =>
+            val Template(parents, self, body) = impl
+            val body1 = trimSyntheticCaseClassMembers(classDef, body)
+            var impl1 = Template(parents, self, body1).copyAttrs(impl)
+            ClassDef(mods, name, params, impl1).copyAttrs(classDef)
+          case moduledef @ ModuleDef(mods, name, impl) =>
+            val Template(parents, self, body) = impl
+            val body1 = trimSyntheticCaseClassMembers(moduledef, body)
+            var impl1 = Template(parents, self, body1).copyAttrs(impl)
+            ModuleDef(mods, name, impl1).copyAttrs(moduledef)
+          case template @ Template(parents, self, body) =>
+            val body1 = trimSyntheticCaseClassCompanions(body)
+            Template(parents, self, body1).copyAttrs(template)
+          case block @ Block(stats, expr) =>
+            val stats1 = trimSyntheticCaseClassCompanions(stats)
+            Block(stats1, expr).copyAttrs(block)
+          case valdef @ ValDef(mods, name, tpt, rhs) if valdef.symbol.isLazy =>
+            if (reifyDebug) println("dropping $lzy in lazy val's name: " + tree)
+            val name1 = if (name endsWith nme.LAZY_LOCAL) name dropRight nme.LAZY_LOCAL.length else name
+            ValDef(mods, name1, tpt, rhs).copyAttrs(valdef)
+          case unapply @ UnApply(fun, args) =>
+            def extractExtractor(tree: Tree): Tree = {
+              val Apply(fun, args) = tree
+              args match {
+                case List(Ident(special)) if special == nme.SELECTOR_DUMMY =>
+                  val Select(extractor, flavor) = fun
+                  assert(flavor == nme.unapply || flavor == nme.unapplySeq)
+                  extractor
+                case _ =>
+                  extractExtractor(fun)
+              }
+            }
+
+            if (reifyDebug) println("unapplying unapply: " + tree)
+            val fun1 = extractExtractor(fun)
+            Apply(fun1, args).copyAttrs(unapply)
+          case _ =>
+            tree
+        }
+      }
+
       /** Reify a tree */
-      private def reifyTree(tree: Tree): Tree = {
-        def reifyDefault(tree: Tree) =
-          reifyProduct(tree)
+      private def reifyTree(tree0: Tree): Tree = {
+        val tree = trimTree(tree0)
 
         var rtree = tree match {
           case tree if tree.isErroneous =>
@@ -311,29 +404,24 @@ trait Reifiers { self: Global =>
             } else reifyFree(tree)
           case tt: TypeTree if (tt.tpe != null) =>
             reifyTypeTree(tt)
-          case ta @ TypeApply(hk, ts) =>
-            def isErased(tt: TypeTree) = tt.tpe != null && definedInLiftedCode(tt.tpe) && tt.original == null
-            val discard = ts collect { case tt: TypeTree => tt } exists isErased
-            if (reifyDebug && discard) println("discarding TypeApply: " + tree)
-            if (discard) reifyTree(hk) else reifyDefault(ta)
           case Literal(constant @ Constant(tpe: Type)) if boundSyms exists (tpe contains _) =>
             CannotReifyClassOfBoundType(tree, tpe)
           case Literal(constant @ Constant(sym: Symbol)) if boundSyms contains sym =>
             CannotReifyClassOfBoundEnum(tree, constant.tpe)
           case tree if tree.isDef =>
             if (reifyDebug) println("boundSym: %s of type %s".format(tree.symbol, (tree.productIterator.toList collect { case tt: TypeTree => tt } headOption).getOrElse(TypeTree(tree.tpe))))
-            // registerReifiableSymbol(tree.symbol)
             boundSyms += tree.symbol
 
-            if (tree.symbol.sourceModule != NoSymbol) {
-              if (reifyDebug) println("boundSym (sourceModule): " + tree.symbol.sourceModule)
-              boundSyms += tree.symbol.sourceModule
-            }
-
-            if (tree.symbol.moduleClass != NoSymbol) {
-              if (reifyDebug) println("boundSym (moduleClass): " + tree.symbol.moduleClass)
-              boundSyms += tree.symbol.moduleClass
-            }
+            bindRelatedSymbol(tree.symbol.sourceModule, "sourceModule")
+            bindRelatedSymbol(tree.symbol.moduleClass, "moduleClass")
+            bindRelatedSymbol(tree.symbol.companionClass, "companionClass")
+            bindRelatedSymbol(tree.symbol.companionModule, "companionModule")
+            Some(tree.symbol) collect { case termSymbol: TermSymbol => bindRelatedSymbol(termSymbol.referenced, "referenced") }
+            def bindRelatedSymbol(related: Symbol, name: String): Unit =
+              if (related != null && related != NoSymbol) {
+                if (reifyDebug) println("boundSym (" + name + "): " + related)
+                boundSyms += related
+              }
 
             val prefix = tree.productPrefix
             val elements = (tree.productIterator map {
@@ -354,7 +442,7 @@ trait Reifiers { self: Global =>
             }).toList
             reifyProduct(prefix, elements)
           case _ =>
-            reifyDefault(tree)
+            reifyProduct(tree)
         }
 
         // usually we don't reify symbols/types, because they can be re-inferred during subsequent reflective compilation
@@ -396,10 +484,8 @@ trait Reifiers { self: Global =>
        *
        *  This workaround worked surprisingly well and allowed me to fix several important reification bugs, until the abstraction has leaked.
        *  Suddenly I found out that in certain contexts original trees do not contain symbols, but are just parser trees.
-       *  To the moment I know two such situations:
-       *  1) Unapplies: https://issues.scala-lang.org/browse/SI-5273?focusedCommentId=56057#comment-56057
-       *  2) Annotations: typedAnnotations does not typecheck the annotation in-place, but rather creates new trees and typechecks them, so the original remains symless
-       *  3) <sigh, what will I discover next?>
+       *  To the moment I know only one such situation: typedAnnotations does not typecheck the annotation in-place, but rather creates new trees and typechecks them, so the original remains symless.
+       *  This is laboriously worked around in the code below. I hope this will be the only workaround in this department.
        */
       private def reifyTypeTree(tt: TypeTree): Tree = {
         if (definedInLiftedCode(tt.tpe)) {
@@ -441,14 +527,15 @@ trait Reifiers { self: Global =>
           }
         } else {
           var rtt = mirrorCall(nme.TypeTree, reifyType(tt.tpe))
-          // @xeno.by: originals get typechecked during subsequent reflective compilation, which leads to subtle bugs
-          // https://issues.scala-lang.org/browse/SI-5273?focusedCommentId=56057#comment-56057
-          // until this is somehow sorted out, I disable reification of originals
-          // if (tt.original != null) {
-          //   val setOriginal = Select(rtt, newTermName("setOriginal"))
-          //   val reifiedOriginal = reify(tt.original)
-          //   rtt = Apply(setOriginal, List(reifiedOriginal))
-          // }
+          // @xeno.by: temporarily disabling reification of originals
+          // subsequent reflective compilation will try to typecheck them
+          // and this means that the reifier has to do additional efforts to ensure that this will succeed
+          // additional efforts + no clear benefit = will be implemented later
+//          if (tt.original != null) {
+//            val setOriginal = Select(rtt, newTermName("setOriginal"))
+//            val reifiedOriginal = reify(tt.original)
+//            rtt = Apply(setOriginal, List(reifiedOriginal))
+//          }
           rtt
         }
       }
diff --git a/src/compiler/scala/tools/nsc/ast/parser/Scanners.scala b/src/compiler/scala/tools/nsc/ast/parser/Scanners.scala
index dae264fffe..f712c7411f 100644
--- a/src/compiler/scala/tools/nsc/ast/parser/Scanners.scala
+++ b/src/compiler/scala/tools/nsc/ast/parser/Scanners.scala
@@ -180,7 +180,7 @@ trait Scanners extends ScannersCommon {
      *  @pre: inStringInterpolation
      */
     @inline private def inMultiLineInterpolation = 
-      sepRegions.tail.nonEmpty && sepRegions.tail.head == STRINGPART    
+      inStringInterpolation && sepRegions.tail.nonEmpty && sepRegions.tail.head == STRINGPART
     
     /** read next token and return last offset
      */
@@ -217,7 +217,9 @@ trait Scanners extends ScannersCommon {
           if (!sepRegions.isEmpty && sepRegions.head == lastToken)
             sepRegions = sepRegions.tail
         case STRINGLIT =>
-          if (inStringInterpolation)
+          if (inMultiLineInterpolation)
+            sepRegions = sepRegions.tail.tail
+          else if (inStringInterpolation)
             sepRegions = sepRegions.tail
         case _ =>
       }
@@ -386,7 +388,7 @@ trait Scanners extends ScannersCommon {
               if (ch == '\"') {
                 nextRawChar()
                 getStringPart(multiLine = true)
-                sepRegions = STRINGLIT :: sepRegions // indicate string part
+                sepRegions = STRINGPART :: sepRegions // indicate string part
                 sepRegions = STRINGLIT :: sepRegions // once more to indicate multi line string part
               } else {
                 token = STRINGLIT