force UTF-8 for slurp and write

1 files changed, 28 insertions, 2 deletions

diff --git a/test/test/CompilerTest.scala b/test/test/CompilerTest.scala
index b56b910bf..d0e4b9a52 100644
--- a/test/test/CompilerTest.scala
+++ b/test/test/CompilerTest.scala
@@ -449,10 +449,36 @@ abstract class CompilerTest {
   /** Recursively copy over source files and directories, excluding extensions
     * that aren't in extensionsToCopy. */
   private def recCopyFiles(sourceFile: Path, dest: Path): Unit = {
-    processFileDir(sourceFile, { sf =>
+    def copyfile(file: SFile, bytewise: Boolean): Unit = {
+      if (bytewise) {
+        val in = file.inputStream()
+        val out = SFile(dest).outputStream()
+        val buffer = new Array[Byte](1024)
+        def loop(available: Int):Unit = {
+          if (available < 0) {()}
+          else {
+            out.write(buffer, 0, available)
+            val read = in.read(buffer)
+            loop(read)
+          }
+        }
+        loop(0)
+        in.close()
+        out.close()
+      } else {
+        try {
+          SFile(dest)(scala.io.Codec.UTF8).writeAll((s"/* !!!!! WARNING: DO NOT MODIFY. Original is at: $file !!!!! */").replace("\\", "/"), file.slurp("UTF-8"))
+        } catch {
+          case unmappable: java.nio.charset.MalformedInputException => 
+            copyfile(file, true) //there are bytes that can't be mapped with UTF-8. Bail and just do a straight byte-wise copy without the warning header.
+        }
+      }
+    }
+
+    processFileDir(sourceFile, { sf => 
       if (extensionsToCopy.contains(sf.extension)) {
         dest.parent.jfile.mkdirs
-        dest.toFile.writeAll(s"/* !!!!! WARNING: DO NOT MODIFY. Original is at: $sf !!!!! */", sf.slurp)
+	      copyfile(sf, false)
       } else {
         log(s"WARNING: ignoring $sf")
       }