Merge pull request #1494 from martijnhoekstra/wintests

[WIP] fix encoding issues

8 files changed, 52 insertions, 18 deletions

diff --git a/test/dotc/tests.scala b/test/dotc/tests.scala
index 9f95a30c1..c4d8085f5 100644
--- a/test/dotc/tests.scala
+++ b/test/dotc/tests.scala
@@ -23,8 +23,7 @@ class tests extends CompilerTest {
   val defaultOutputDir = "./out/"
 
   implicit val defaultOptions = noCheckOptions ++ List(
-      "-Yno-deep-subtypes", "-Yno-double-bindings", "-Yforce-sbt-phases",
-      "-d", defaultOutputDir) ++ {
+      "-Yno-deep-subtypes", "-Yno-double-bindings", "-Yforce-sbt-phases", "-d", defaultOutputDir) ++ {
     if (isRunByJenkins) List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") // should be Ycheck:all, but #725
     else List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef")
   }
@@ -38,6 +37,9 @@ class tests extends CompilerTest {
   val allowDoubleBindings = defaultOptions diff List("-Yno-double-bindings")
   val scala2mode = List("-language:Scala2")
 
+  val explicitUTF8 = List("-encoding", "UTF8")
+  val explicitUTF16 = List("-encoding", "UTF16")
+
   val testsDir      = "./tests/"
   val posDir        = testsDir + "pos/"
   val posSpecialDir = testsDir + "pos-special/"
@@ -95,7 +97,7 @@ class tests extends CompilerTest {
   @Test def pos_overloadedAccess = compileFile(posDir, "overloadedAccess", twice)
   @Test def pos_approximateUnion = compileFile(posDir, "approximateUnion", twice)
   @Test def pos_tailcall = compileDir(posDir, "tailcall", twice)
-  @Test def pos_valueclasses = compileFiles(posDir + "valueclasses/", twice)
+  @Test def pos_valueclasses = compileFiles(posDir + "pos_valueclasses/", twice)
   @Test def pos_nullarify = compileFile(posDir, "nullarify", args = "-Ycheck:nullarify" :: Nil)
   @Test def pos_subtyping = compileFile(posDir, "subtyping", twice)
   @Test def pos_packageObj = compileFile(posDir, "i0239", twice)
@@ -118,6 +120,8 @@ class tests extends CompilerTest {
     compileFile(posSpecialDir, "spec-t5545/S_1")
     compileFile(posSpecialDir, "spec-t5545/S_2")
   }
+  @Test def pos_utf8 = compileFile(posSpecialDir, "utf8encoded", explicitUTF8)
+  @Test def pos_utf16 = compileFile(posSpecialDir, "utf16encoded", explicitUTF16)
 
   @Test def new_all = compileFiles(newDir, twice)
   @Test def repl_all = replFiles(replDir)
diff --git a/test/test/CompilerTest.scala b/test/test/CompilerTest.scala
index 56b9e1099..dea6a30b1 100644
--- a/test/test/CompilerTest.scala
+++ b/test/test/CompilerTest.scala
@@ -228,7 +228,7 @@ abstract class CompilerTest {
 
   private def expectedErrors(filePath: String): List[ErrorsInFile] = expectedErrors(List(filePath))
 
-  private def isNegTest(testPath: String) = testPath.contains(JFile.separator + "neg" + JFile.separator)
+  private def isNegTest(testPath: String) = testPath.contains("/neg/")
 
   private def compileArgs(args: Array[String], expectedErrorsPerFile: List[ErrorsInFile])
       (implicit defaultOptions: List[String]): Unit = {
@@ -413,7 +413,8 @@ abstract class CompilerTest {
     val flags = oldFlags.map(f => if (f == oldOutput) partestOutput else f) ++
       List(s"-classpath $partestOutput") // Required for separate compilation tests
 
-    getExisting(dest).isDifferent(source, flags, nerr) match {
+    val difference = getExisting(dest).isDifferent(source, flags, nerr)
+    difference match {
       case NotExists => copyFiles(source, dest, partestOutput, flags, nerr, kind)
       case ExistsSame => // nothing else to do
       case ExistsDifferent =>
@@ -449,11 +450,37 @@ abstract class CompilerTest {
   /** Recursively copy over source files and directories, excluding extensions
     * that aren't in extensionsToCopy. */
   private def recCopyFiles(sourceFile: Path, dest: Path): Unit = {
-    processFileDir(sourceFile, { sf =>
+
+    def copyfile(file: SFile, bytewise: Boolean): Unit = {
+      if (bytewise) {
+        val in = file.inputStream()
+        val out = SFile(dest).outputStream()
+        val buffer = new Array[Byte](1024)
+        def loop(available: Int):Unit = {
+          if (available < 0) {()}
+          else {
+            out.write(buffer, 0, available)
+            val read = in.read(buffer)
+            loop(read)
+          }
+        }
+        loop(0)
+        in.close()
+        out.close()
+      } else {
+        try {
+          SFile(dest)(scala.io.Codec.UTF8).writeAll((s"/* !!!!! WARNING: DO NOT MODIFY. Original is at: $file !!!!! */").replace("\\", "/"), file.slurp("UTF-8"))
+        } catch {
+          case unmappable: java.nio.charset.MalformedInputException => 
+            copyfile(file, true) //there are bytes that can't be mapped with UTF-8. Bail and just do a straight byte-wise copy without the warning header.
+        }
+      }
+    }
+
+    processFileDir(sourceFile, { sf => 
       if (extensionsToCopy.contains(sf.extension)) {
         dest.parent.jfile.mkdirs
-        dest.toFile.writeAll("/* !!!!! WARNING: DO NOT MODIFY. Original is at: $sf !!!!! */",
-          sf.slurp())
+	      copyfile(sf, false)
       } else {
         log(s"WARNING: ignoring $sf")
       }
@@ -465,7 +492,7 @@ abstract class CompilerTest {
 
   /** Reads the existing files for the given test source if any. */
   private def getExisting(dest: Path): ExistingFiles = {
-    val content: Option[Option[String]] = processFileDir(dest, f => f.safeSlurp, d => Some(""))
+    val content: Option[Option[String]] = processFileDir(dest, f => try Some(f.slurp("UTF8")) catch {case io: java.io.IOException => Some(io.toString())}, d => Some(""))
     if (content.isDefined && content.get.isDefined) {
       val flags = (dest changeExtension "flags").toFile.safeSlurp
       val nerr = (dest changeExtension "nerr").toFile.safeSlurp
@@ -479,7 +506,7 @@ abstract class CompilerTest {
       if (!genSrc.isDefined) {
         NotExists
       } else {
-        val source = processFileDir(sourceFile, { f => f.safeSlurp }, { d => Some("") },
+        val source = processFileDir(sourceFile, { f => try Some(f.slurp("UTF8")) catch {case _: java.io.IOException => None} }, { d => Some("") },
             Some("DPCompilerTest sourceFile doesn't exist: " + sourceFile)).get
         if (source == genSrc) {
           nerr match {
diff --git a/test/test/DottyTest.scala b/test/test/DottyTest.scala
index 15d82c208..4b767b318 100644
--- a/test/test/DottyTest.scala
+++ b/test/test/DottyTest.scala
@@ -23,6 +23,7 @@ class DottyTest /*extends ContextEscapeDetection*/ {
     import base.settings._
     val ctx = base.initialCtx.fresh
     base.initialize()(ctx)
+    ctx.setSetting(ctx.settings.encoding, "UTF8")
     ctx
   }
 /*
diff --git a/test/test/InterfaceEntryPointTest.scala b/test/test/InterfaceEntryPointTest.scala
index b193b5f64..438a9fa47 100644
--- a/test/test/InterfaceEntryPointTest.scala
+++ b/test/test/InterfaceEntryPointTest.scala
@@ -18,7 +18,7 @@ import scala.collection.mutable.ListBuffer
  */
 class InterfaceEntryPointTest {
   @Test def runCompilerFromInterface = {
-    val sources = List("./tests/pos/HelloWorld.scala")
+    val sources = List("./tests/pos/HelloWorld.scala").map(p => new java.io.File(p).getPath())
     val args = sources ++ List("-d", "./out/")
 
     val mainClass = Class.forName("dotty.tools.dotc.Main")
diff --git a/test/test/OtherEntryPointsTest.scala b/test/test/OtherEntryPointsTest.scala
index 0186b357b..5f8681d38 100644
--- a/test/test/OtherEntryPointsTest.scala
+++ b/test/test/OtherEntryPointsTest.scala
@@ -17,7 +17,7 @@ import scala.collection.mutable.ListBuffer
  */
 class OtherEntryPointsTest {
   @Test def runCompiler = {
-    val sources = List("./tests/pos/HelloWorld.scala")
+    val sources = List("./tests/pos/HelloWorld.scala").map(p => new java.io.File(p).getPath())
     val args = sources ++ List("-d", "./out/")
 
     val reporter = new CustomReporter
@@ -31,7 +31,7 @@ class OtherEntryPointsTest {
   }
 
   @Test def runCompilerWithContext = {
-    val sources = List("./tests/pos/HelloWorld.scala")
+    val sources = List("./tests/pos/HelloWorld.scala").map(p => new java.io.File(p).getPath())
     val args = sources ++ List("-d", "./out/")
 
     val reporter = new CustomReporter
diff --git a/test/test/ParserTest.scala b/test/test/ParserTest.scala
index 524be272d..f66dbf55d 100644
--- a/test/test/ParserTest.scala
+++ b/test/test/ParserTest.scala
@@ -4,6 +4,7 @@ import scala.reflect.io._
 import dotty.tools.dotc.util._
 import dotty.tools.dotc.core._
 import dotty.tools.dotc.parsing._
+import scala.io.Codec
 import Tokens._, Parsers._
 import dotty.tools.dotc.ast.untpd._
 import org.junit.Test
@@ -23,7 +24,7 @@ class ParserTest extends DottyTest {
 
   def parse(file: PlainFile): Tree = {
     //println("***** parsing " + file)
-    val source = new SourceFile(file)
+    val source = new SourceFile(file, Codec.UTF8)
     val parser = new Parser(source)
     val tree = parser.parse()
     parsed += 1
diff --git a/test/test/ScannerTest.scala b/test/test/ScannerTest.scala
index f8f09ff6f..5ff9bba0c 100644
--- a/test/test/ScannerTest.scala
+++ b/test/test/ScannerTest.scala
@@ -1,6 +1,7 @@
 package test
 
 import scala.reflect.io._
+import scala.io.Codec
 import dotty.tools.dotc.util._
 import dotty.tools.dotc.parsing._
 import Tokens._, Scanners._
@@ -16,8 +17,8 @@ class ScannerTest extends DottyTest {
   def scan(name: String): Unit = scan(new PlainFile(name))
 
   def scan(file: PlainFile): Unit = {
-    println("***** scanning " + file)
-    val source = new SourceFile(file)
+    //println("***** scanning " + file)
+    val source = new SourceFile(file, Codec.UTF8)
     val scanner = new Scanner(source)
     var i = 0
     while (scanner.token != EOF) {
diff --git a/test/test/TestREPL.scala b/test/test/TestREPL.scala
index 1ba456ce2..9867cb4ec 100644
--- a/test/test/TestREPL.scala
+++ b/test/test/TestREPL.scala
@@ -34,7 +34,7 @@ class TestREPL(script: String) extends REPL {
           while (lines.hasNext && lines.head.startsWith(continuationPrompt)) {
             val continued = lines.next
             output.println(continued)
-            buf append "\n"
+            buf append System.lineSeparator()
             buf append continued.drop(continuationPrompt.length)
           }
           buf.toString
@@ -49,7 +49,7 @@ class TestREPL(script: String) extends REPL {
     out.close()
     val printed = out.toString
     val transcript = printed.drop(printed.indexOf(config.prompt))
-    if (transcript.toString != script) {
+    if (transcript.toString.lines.toList != script.lines.toList) {
       println("input differs from transcript:")
       println(transcript)
       assert(false)