[backport] Accessibility checks for methods with an InvokeDynamic instruction

Implements the necessary tests to check if a method with an
InvokeDynamic instruction can be inlined into a destination class.

Only InvokeDynamic instructions with LambdaMetaFactory as bootstrap
methods can be inlined. The accessibility checks cannot be implemented
generically, because it depends on what the bootstrap method is doing.
In particular, the bootstrap method receives a Lookup object as
argument which can be used to access private methods of the class
where the InvokeDynamic method is located.

A comment in the inliner explains the details.

2 files changed, 68 insertions, 4 deletions

diff --git a/src/compiler/scala/tools/nsc/backend/jvm/BackendReporting.scala b/src/compiler/scala/tools/nsc/backend/jvm/BackendReporting.scala
index 4fc05cafdc..4eb24d13e3 100644
--- a/src/compiler/scala/tools/nsc/backend/jvm/BackendReporting.scala
+++ b/src/compiler/scala/tools/nsc/backend/jvm/BackendReporting.scala
@@ -1,7 +1,7 @@
 package scala.tools.nsc
 package backend.jvm
 
-import scala.tools.asm.tree.{AbstractInsnNode, MethodNode}
+import scala.tools.asm.tree.{InvokeDynamicInsnNode, AbstractInsnNode, MethodNode}
 import scala.tools.nsc.backend.jvm.BTypes.InternalName
 import scala.reflect.internal.util.Position
 import scala.tools.nsc.settings.ScalaSettings
@@ -246,6 +246,11 @@ object BackendReporting {
   case class ResultingMethodTooLarge(calleeDeclarationClass: InternalName, name: String, descriptor: String,
                                      callsiteClass: InternalName, callsiteName: String, callsiteDesc: String) extends CannotInlineWarning
 
+  case object UnknownInvokeDynamicInstruction extends OptimizerWarning {
+    override def toString = "The callee contains an InvokeDynamic instruction with an unknown bootstrap method (not a LambdaMetaFactory)."
+    def emitWarning(settings: ScalaSettings): Boolean = settings.YoptWarningEmitAtInlineFailed
+  }
+
   /**
    * Used in `rewriteClosureApplyInvocations` when a closure apply callsite cannot be rewritten
    * to the closure body method.
diff --git a/src/compiler/scala/tools/nsc/backend/jvm/opt/Inliner.scala b/src/compiler/scala/tools/nsc/backend/jvm/opt/Inliner.scala
index e8e848161c..4a022b7bc4 100644
--- a/src/compiler/scala/tools/nsc/backend/jvm/opt/Inliner.scala
+++ b/src/compiler/scala/tools/nsc/backend/jvm/opt/Inliner.scala
@@ -9,6 +9,7 @@ package opt
 
 import scala.annotation.tailrec
 import scala.tools.asm
+import asm.Handle
 import asm.Opcodes._
 import asm.tree._
 import scala.collection.convert.decorateAsScala._
@@ -687,9 +688,67 @@ class Inliner[BT <: BTypes](val btypes: BT) {
           }
         }
 
-      case ivd: InvokeDynamicInsnNode =>
-        // TODO @lry check necessary conditions to inline an indy, instead of giving up
-        Right(false)
+      case indy: InvokeDynamicInsnNode =>
+        // an indy instr points to a "call site specifier" (CSP) [1]
+        //  - a reference to a bootstrap method [2]
+        //    - bootstrap method name
+        //    - references to constant arguments, which can be:
+        //      - constant (string, long, int, float, double)
+        //      - class
+        //      - method type (without name)
+        //      - method handle
+        //  - a method name+type
+        //
+        // execution [3]
+        //  - resolve the CSP, yielding the boostrap method handle, the static args and the name+type
+        //    - resolution entails accessibility checking [4]
+        //  - execute the `invoke` method of the boostrap method handle (which is signature polymorphic, check its javadoc)
+        //    - the descriptor for the call is made up from the actual arguments on the stack:
+        //      - the first parameters are "MethodHandles.Lookup, String, MethodType", then the types of the constant arguments,
+        //      - the return type is CallSite
+        //    - the values for the call are
+        //      - the bootstrap method handle of the CSP is the receiver
+        //      - the Lookup object for the class in which the callsite occurs (obtained as through calling MethodHandles.lookup())
+        //      - the method name of the CSP
+        //      - the method type of the CSP
+        //      - the constants of the CSP (primitives are not boxed)
+        //  - the resulting `CallSite` object
+        //    - has as `type` the method type of the CSP
+        //    - is popped from the operand stack
+        //  - the `invokeExact` method (signature polymorphic!) of the `target` method handle of the CallSite is invoked
+        //    - the method descriptor is that of the CSP
+        //    - the receiver is the target of the CallSite
+        //    - the other argument values are those that were on the operand stack at the indy instruction (indyLambda: the captured values)
+        //
+        // [1] http://docs.oracle.com/javase/specs/jvms/se8/html/jvms-4.html#jvms-4.4.10
+        // [2] http://docs.oracle.com/javase/specs/jvms/se8/html/jvms-4.html#jvms-4.7.23
+        // [3] http://docs.oracle.com/javase/specs/jvms/se8/html/jvms-6.html#jvms-6.5.invokedynamic
+        // [4] http://docs.oracle.com/javase/specs/jvms/se8/html/jvms-5.html#jvms-5.4.3
+
+        // We cannot generically check if an `invokedynamic` instruction can be safely inlined into
+        // a different class, that depends on the bootstrap method. The Lookup object passed to the
+        // bootstrap method is a capability to access private members of the callsite class. We can
+        // only move the invokedynamic to a new class if we know that the bootstrap method doesn't
+        // use this capability for otherwise non-accessible members.
+        // In the case of indyLambda, it depends on the visibility of the implMethod handle. If
+        // the implMethod is public, lambdaMetaFactory doesn't use the Lookup object's extended
+        // capability, and we can safely inline the instruction into a different class.
+
+        if (destinationClass == calleeDeclarationClass) {
+          Right(true) // within the same class, any indy instruction can be inlined
+        } else if (closureOptimizer.isClosureInstantiation(indy)) {
+          val implMethod = indy.bsmArgs(1).asInstanceOf[Handle] // safe, checked in isClosureInstantiation
+          val methodRefClass = classBTypeFromParsedClassfile(implMethod.getOwner)
+          for {
+            (methodNode, methodDeclClassNode) <- byteCodeRepository.methodNode(methodRefClass.internalName, implMethod.getName, implMethod.getDesc): Either[OptimizerWarning, (MethodNode, InternalName)]
+            methodDeclClass                   =  classBTypeFromParsedClassfile(methodDeclClassNode)
+            res                               <- memberIsAccessible(methodNode.access, methodDeclClass, methodRefClass, destinationClass)
+          } yield {
+            res
+          }
+        } else {
+          Left(UnknownInvokeDynamicInstruction)
+        }
 
       case ci: LdcInsnNode => ci.cst match {
         case t: asm.Type => classIsAccessible(bTypeForDescriptorOrInternalNameFromClassfile(t.getInternalName), destinationClass)