isSubtype: try to dealias TypeRefs before recursively checking the prefixes

As demonstrated by tests/pos/hk-deep-subtype.scala, we can avoid some
deep subtype recursions that result in stack overflows by doing this.

Fix #943.

1 files changed, 15 insertions, 0 deletions

diff --git a/tests/pos/hk-deep-subtype.scala b/tests/pos/hk-deep-subtype.scala
new file mode 100644
index 000000000..202329a79
--- /dev/null
+++ b/tests/pos/hk-deep-subtype.scala
@@ -0,0 +1,15 @@
+// Minimized from scala.collection.generic.GenTraversableFactory plus dependencies
+import scala.annotation.unchecked.uncheckedVariance
+
+trait GT[A] extends GTT[A, GT]
+
+trait HNB[B]
+trait GTT[+C, DD[X] <: GT[X]] extends HNB[DD[C] @uncheckedVariance] // Can be any annotation and still crash
+
+class GTF[EE[X] <: GT[X] with GTT[X, EE]]
+{
+  def foo[F]: EE[F] = ???
+  def bar[G](f: G): EE[G] = ???
+
+  def tabulate: EE[EE[Int]] = bar(foo)
+}