diff options
Diffstat (limited to 'test-suite/src/test/scala/scala/scalajs/testsuite/niocharset')
6 files changed, 887 insertions, 0 deletions
diff --git a/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/BaseCharsetTest.scala b/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/BaseCharsetTest.scala new file mode 100644 index 0000000..adda838 --- /dev/null +++ b/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/BaseCharsetTest.scala @@ -0,0 +1,234 @@ +/* __ *\ +** ________ ___ / / ___ __ ____ Scala.js Test Suite ** +** / __/ __// _ | / / / _ | __ / // __/ (c) 2013, LAMP/EPFL ** +** __\ \/ /__/ __ |/ /__/ __ |/_// /_\ \ http://scala-js.org/ ** +** /____/\___/_/ |_/____/_/ | |__/ /____/ ** +** |/____/ ** +\* */ +package scala.scalajs.testsuite.niocharset + +import scala.language.implicitConversions + +import scala.util._ + +import java.nio._ +import java.nio.charset._ + +import scala.scalajs.js +import js.JSConverters._ + +import org.scalajs.jasminetest.JasmineTest + +class BaseCharsetTest(val charset: Charset) extends JasmineTest { + import BaseCharsetTest._ + + protected val AllErrorActions = Seq( + CodingErrorAction.IGNORE, + CodingErrorAction.REPLACE, + CodingErrorAction.REPORT) + + protected val ReportActions = Seq( + CodingErrorAction.REPORT) + + protected def testDecode(in: ByteBuffer)( + outParts: OutPart[CharBuffer]*): Unit = { + + def testOneConfig(malformedAction: CodingErrorAction, + unmappableAction: CodingErrorAction): Unit = { + + val decoder = charset.newDecoder() + decoder.onMalformedInput(malformedAction) + decoder.onUnmappableCharacter(unmappableAction) + + val actualTry = Try { + in.mark() + val buf = + try decoder.decode(in) + finally in.reset() + val actualChars = new Array[Char](buf.remaining()) + buf.get(actualChars) + actualChars + } + + val expectedTry = Try { + val expectedChars = Array.newBuilder[Char] + outParts foreach { + case BufferPart(buf) => + val bufArray = new Array[Char](buf.remaining) + buf.mark() + try buf.get(bufArray) + finally buf.reset() + expectedChars ++= bufArray + case Malformed(len) => + malformedAction match { + case CodingErrorAction.IGNORE => + case CodingErrorAction.REPLACE => + expectedChars ++= decoder.replacement() + case CodingErrorAction.REPORT => + throw new MalformedInputException(len) + } + case Unmappable(len) => + unmappableAction match { + case CodingErrorAction.IGNORE => + case CodingErrorAction.REPLACE => + expectedChars ++= decoder.replacement() + case CodingErrorAction.REPORT => + throw new UnmappableCharacterException(len) + } + } + expectedChars.result() + } + + (actualTry, expectedTry) match { + case (Failure(actualEx: MalformedInputException), + Failure(expectedEx: MalformedInputException)) => + expect(actualEx.getInputLength()).toEqual(expectedEx.getInputLength()) + + case (Failure(actualEx: UnmappableCharacterException), + Failure(expectedEx: UnmappableCharacterException)) => + expect(actualEx.getInputLength()).toEqual(expectedEx.getInputLength()) + + case (Success(actualChars), Success(expectedChars)) => + expect(actualChars.map(_.toInt).toJSArray).toEqual( + expectedChars.map(_.toInt).toJSArray) + + case _ => + // For the error message + expect(actualTry.asInstanceOf[js.Any]).toBe( + expectedTry.asInstanceOf[js.Any]) + } + } + + val hasAnyMalformed = outParts.exists(_.isInstanceOf[Malformed]) + val hasAnyUnmappable = outParts.exists(_.isInstanceOf[Unmappable]) + + for { + malformedAction <- if (hasAnyMalformed) AllErrorActions else ReportActions + unmappableAction <- if (hasAnyUnmappable) AllErrorActions else ReportActions + } { + testOneConfig(malformedAction, unmappableAction) + } + } + + protected def testEncode(in: CharBuffer)( + outParts: OutPart[ByteBuffer]*): Unit = { + + def testOneConfig(malformedAction: CodingErrorAction, + unmappableAction: CodingErrorAction): Unit = { + + val encoder = charset.newEncoder() + encoder.onMalformedInput(malformedAction) + encoder.onUnmappableCharacter(unmappableAction) + + val actualTry = Try { + in.mark() + val buf = + try encoder.encode(in) + finally in.reset() + val actualBytes = new Array[Byte](buf.remaining()) + buf.get(actualBytes) + actualBytes + } + + val expectedTry = Try { + val expectedBytes = Array.newBuilder[Byte] + outParts foreach { + case BufferPart(buf) => + val bufArray = new Array[Byte](buf.remaining) + buf.mark() + try buf.get(bufArray) + finally buf.reset() + expectedBytes ++= bufArray + case Malformed(len) => + malformedAction match { + case CodingErrorAction.IGNORE => + case CodingErrorAction.REPLACE => + expectedBytes ++= encoder.replacement() + case CodingErrorAction.REPORT => + throw new MalformedInputException(len) + } + case Unmappable(len) => + unmappableAction match { + case CodingErrorAction.IGNORE => + case CodingErrorAction.REPLACE => + expectedBytes ++= encoder.replacement() + case CodingErrorAction.REPORT => + throw new UnmappableCharacterException(len) + } + } + expectedBytes.result() + } + + (actualTry, expectedTry) match { + case (Failure(actualEx: MalformedInputException), + Failure(expectedEx: MalformedInputException)) => + expect(actualEx.getInputLength()).toEqual(expectedEx.getInputLength()) + + case (Failure(actualEx: UnmappableCharacterException), + Failure(expectedEx: UnmappableCharacterException)) => + expect(actualEx.getInputLength()).toEqual(expectedEx.getInputLength()) + + case (Success(actualBytes), Success(expectedBytes)) => + expect(actualBytes.toJSArray).toEqual(expectedBytes.toJSArray) + + case _ => + // For the error message + expect(actualTry.asInstanceOf[js.Any]).toBe( + expectedTry.asInstanceOf[js.Any]) + } + } + + val hasAnyMalformed = outParts.exists(_.isInstanceOf[Malformed]) + val hasAnyUnmappable = outParts.exists(_.isInstanceOf[Unmappable]) + + for { + malformedAction <- if (hasAnyMalformed) AllErrorActions else ReportActions + unmappableAction <- if (hasAnyUnmappable) AllErrorActions else ReportActions + } { + testOneConfig(malformedAction, unmappableAction) + } + } +} + +object BaseCharsetTest { + sealed abstract class OutPart[+BufferType <: Buffer] + final case class BufferPart[BufferType <: Buffer](buf: BufferType) extends OutPart[BufferType] + final case class Malformed(length: Int) extends OutPart[Nothing] + final case class Unmappable(length: Int) extends OutPart[Nothing] + + object OutPart { + implicit def fromBuffer[BufferType <: Buffer](buf: BufferType): BufferPart[BufferType] = + BufferPart(buf) + } + + implicit class Interpolators(val sc: StringContext) extends AnyVal { + def bb(args: Any*): ByteBuffer = { + val strings = sc.parts.iterator + val expressions = args.iterator + val buf = Array.newBuilder[Byte] + + def appendStr(s: String): Unit = { + val s1 = s.replace(" ", "") + require(s1.length % 2 == 0) + for (i <- 0 until s1.length by 2) + buf += java.lang.Integer.parseInt(s1.substring(i, i+2), 16).toByte + } + + appendStr(strings.next()) + while (strings.hasNext) { + expressions.next() match { + case b: Byte => buf += b + case bytes: Array[Byte] => buf ++= bytes + case bytes: Seq[_] => + buf ++= bytes.map(_.asInstanceOf[Number].byteValue()) + } + appendStr(strings.next()) + } + + ByteBuffer.wrap(buf.result()) + } + + def cb(args: Any*): CharBuffer = + CharBuffer.wrap(sc.s(args: _*)) + } +} diff --git a/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/CharsetTest.scala b/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/CharsetTest.scala new file mode 100644 index 0000000..15b0150 --- /dev/null +++ b/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/CharsetTest.scala @@ -0,0 +1,74 @@ +/* __ *\ +** ________ ___ / / ___ __ ____ Scala.js Test Suite ** +** / __/ __// _ | / / / _ | __ / // __/ (c) 2013, LAMP/EPFL ** +** __\ \/ /__/ __ |/ /__/ __ |/_// /_\ \ http://scala-js.org/ ** +** /____/\___/_/ |_/____/_/ | |__/ /____/ ** +** |/____/ ** +\* */ +package scala.scalajs.testsuite.niocharset + +import scala.language.implicitConversions + +import java.nio._ +import java.nio.charset._ + +import scala.scalajs.js +import scala.scalajs.niocharset.StandardCharsets._ + +import org.scalajs.jasminetest.JasmineTest + +object CharsetTest extends JasmineTest { + implicit def charsetAsJSAny(charset: Charset): js.Any = + charset.asInstanceOf[js.Any] + + describe("java.nio.charset.Charset") { + it("defaultCharset") { + expect(Charset.defaultCharset()).toBe(UTF_8) + } + + it("forName") { + expect(Charset.forName("ISO-8859-1")).toBe(ISO_8859_1) + expect(Charset.forName("Iso8859-1")).toBe(ISO_8859_1) + expect(Charset.forName("iso_8859_1")).toBe(ISO_8859_1) + expect(Charset.forName("LaTin1")).toBe(ISO_8859_1) + expect(Charset.forName("l1")).toBe(ISO_8859_1) + + expect(Charset.forName("US-ASCII")).toBe(US_ASCII) + expect(Charset.forName("Default")).toBe(US_ASCII) + + expect(Charset.forName("UTF-8")).toBe(UTF_8) + expect(Charset.forName("utf-8")).toBe(UTF_8) + expect(Charset.forName("UtF8")).toBe(UTF_8) + expect(Charset.forName("UTF_8")).toBe(UTF_8) + expect(Charset.forName("UTF-8")).toBe(UTF_8) + + expect(Charset.forName("UTF-16BE")).toBe(UTF_16BE) + expect(Charset.forName("Utf_16BE")).toBe(UTF_16BE) + expect(Charset.forName("UnicodeBigUnmarked")).toBe(UTF_16BE) + + expect(Charset.forName("UTF-16le")).toBe(UTF_16LE) + expect(Charset.forName("Utf_16le")).toBe(UTF_16LE) + expect(Charset.forName("UnicodeLittleUnmarked")).toBe(UTF_16LE) + + expect(Charset.forName("UTF-16")).toBe(UTF_16) + expect(Charset.forName("Utf_16")).toBe(UTF_16) + expect(Charset.forName("unicode")).toBe(UTF_16) + expect(Charset.forName("UnicodeBig")).toBe(UTF_16) + + expect(() => Charset.forName("this-charset-does-not-exist")).toThrow + } + + it("isSupported") { + expect(Charset.isSupported("ISO-8859-1")).toBeTruthy + expect(Charset.isSupported("US-ASCII")).toBeTruthy + expect(Charset.isSupported("Default")).toBeTruthy + expect(Charset.isSupported("utf-8")).toBeTruthy + expect(Charset.isSupported("UnicodeBigUnmarked")).toBeTruthy + expect(Charset.isSupported("Utf_16le")).toBeTruthy + expect(Charset.isSupported("UTF-16")).toBeTruthy + expect(Charset.isSupported("unicode")).toBeTruthy + + expect(Charset.isSupported("this-charset-does-not-exist")).toBeFalsy + } + } +} diff --git a/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/Latin1Test.scala b/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/Latin1Test.scala new file mode 100644 index 0000000..f6ddc36 --- /dev/null +++ b/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/Latin1Test.scala @@ -0,0 +1,91 @@ +/* __ *\ +** ________ ___ / / ___ __ ____ Scala.js Test Suite ** +** / __/ __// _ | / / / _ | __ / // __/ (c) 2013, LAMP/EPFL ** +** __\ \/ /__/ __ |/ /__/ __ |/_// /_\ \ http://scala-js.org/ ** +** /____/\___/_/ |_/____/_/ | |__/ /____/ ** +** |/____/ ** +\* */ +package scala.scalajs.testsuite.niocharset + +import java.nio._ +import java.nio.charset._ + +import scala.scalajs.niocharset.StandardCharsets + +import BaseCharsetTest._ + +object Latin1Test extends BaseCharsetTest(StandardCharsets.ISO_8859_1) { + describe("ISO-8859-1") { + it("decode") { + // Simple tests + + testDecode(bb"48 65 6c 6c 6f")(cb"Hello") + testDecode(bb"42 6f 6e 6a 6f 75 72")(cb"Bonjour") + + testDecode(bb"00 01 0a 10 20")(cb"\u0000\u0001\u000a\u0010 ") + testDecode(bb"7f 7f")(cb"\u007f\u007f") + + testDecode(bb"c8 e5 ec ec ef")(cb"Èåììï") + testDecode(bb"c2 ef ee ea ef f5 f2")(cb"Âïîêïõò") + + testDecode(bb"80 81 8a 90 a0")(cb"\u0080\u0081\u008a\u0090\u00a0") + testDecode(bb"ff ff")(cb"ÿÿ") + } + + it("encode") { + // Simple tests + + testEncode(cb"Hello")(bb"48 65 6c 6c 6f") + testEncode(cb"Bonjour")(bb"42 6f 6e 6a 6f 75 72") + + testEncode(cb"\u0000\u0001\u000a\u0010 ")(bb"00 01 0a 10 20") + testEncode(cb"\u007f\u007f")(bb"7f 7f") + + testEncode(cb"Èåììï")(bb"c8 e5 ec ec ef") + testEncode(cb"Âïîêïõò")(bb"c2 ef ee ea ef f5 f2") + + testEncode(cb"\u0080\u0081\u008a\u0090\u00a0")(bb"80 81 8a 90 a0") + testEncode(cb"ÿÿ")(bb"ff ff") + + // Unmappable characters + + testEncode(cb"\u0100")(Unmappable(1)) + testEncode(cb"\u07ff")(Unmappable(1)) + testEncode(cb"\ue000")(Unmappable(1)) + testEncode(cb"\uffff")(Unmappable(1)) + testEncode(cb"\ud835\udcd7")(Unmappable(2)) + + testEncode(cb"\u0100A")(Unmappable(1), bb"41") + testEncode(cb"\u07ffA")(Unmappable(1), bb"41") + testEncode(cb"\ue000A")(Unmappable(1), bb"41") + testEncode(cb"\uffffA")(Unmappable(1), bb"41") + testEncode(cb"\ud835\udcd7A")(Unmappable(2), bb"41") + + // Single UTF-16 surrogates + testEncode(cb"\ud800")(Malformed(1)) + testEncode(cb"\udaff")(Malformed(1)) + testEncode(cb"\udb80")(Malformed(1)) + testEncode(cb"\udbff")(Malformed(1)) + testEncode(cb"\udc00")(Malformed(1)) + testEncode(cb"\udf80")(Malformed(1)) + testEncode(cb"\udfff")(Malformed(1)) + + // High UTF-16 surrogates not followed by low surrogates + testEncode(cb"\ud800A")(Malformed(1), bb"41") + testEncode(cb"\ud800\ud800")(Malformed(1), Malformed(1)) + testEncode(cb"\ud800\ud835\udcd7")(Malformed(1), Unmappable(2)) + testEncode(cb"\udbffA")(Malformed(1), bb"41") + testEncode(cb"\udbff\udb8f")(Malformed(1), Malformed(1)) + testEncode(cb"\udbff\ud835\udcd7")(Malformed(1), Unmappable(2)) + } + + it("isLegalReplacement") { + val encoder = charset.newEncoder + expect(encoder.isLegalReplacement(Array(0x00.toByte))).toBeTruthy + expect(encoder.isLegalReplacement(Array(0x41.toByte))).toBeTruthy + expect(encoder.isLegalReplacement(Array('?'.toByte))).toBeTruthy + expect(encoder.isLegalReplacement(Array(0x80.toByte))).toBeTruthy + expect(encoder.isLegalReplacement(Array(0xff.toByte))).toBeTruthy + } + } +} diff --git a/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/USASCIITest.scala b/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/USASCIITest.scala new file mode 100644 index 0000000..2352d3e --- /dev/null +++ b/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/USASCIITest.scala @@ -0,0 +1,93 @@ +/* __ *\ +** ________ ___ / / ___ __ ____ Scala.js Test Suite ** +** / __/ __// _ | / / / _ | __ / // __/ (c) 2013, LAMP/EPFL ** +** __\ \/ /__/ __ |/ /__/ __ |/_// /_\ \ http://scala-js.org/ ** +** /____/\___/_/ |_/____/_/ | |__/ /____/ ** +** |/____/ ** +\* */ +package scala.scalajs.testsuite.niocharset + +import java.nio._ +import java.nio.charset._ + +import scala.scalajs.niocharset.StandardCharsets + +import BaseCharsetTest._ + +object USASCIITest extends BaseCharsetTest(StandardCharsets.US_ASCII) { + describe("US-ASCII") { + it("decode") { + // Simple tests + + testDecode(bb"48 65 6c 6c 6f")(cb"Hello") + testDecode(bb"42 6f 6e 6a 6f 75 72")(cb"Bonjour") + + testDecode(bb"00 01 0a 10 20")(cb"\u0000\u0001\u000a\u0010 ") + testDecode(bb"7f 7f")(cb"\u007f\u007f") + + // Bit 7 is ignored, giving the same results as above + + testDecode(bb"c8 e5 ec ec ef")(cb"Hello") + testDecode(bb"c2 ef ee ea ef f5 f2")(cb"Bonjour") + + testDecode(bb"80 81 8a 90 a0")(cb"\u0000\u0001\u000a\u0010 ") + testDecode(bb"ff ff")(cb"\u007f\u007f") + } + + it("encode") { + // Simple tests + + testEncode(cb"Hello")(bb"48 65 6c 6c 6f") + testEncode(cb"Bonjour")(bb"42 6f 6e 6a 6f 75 72") + + testEncode(cb"\u0000\u0001\u000a\u0010 ")(bb"00 01 0a 10 20") + testEncode(cb"\u007f\u007f")(bb"7f 7f") + + // Unmappable characters + + testEncode(cb"é")(Unmappable(1)) + testEncode(cb"\u0080")(Unmappable(1)) + testEncode(cb"\u00ff")(Unmappable(1)) + testEncode(cb"\u0100")(Unmappable(1)) + testEncode(cb"\u07ff")(Unmappable(1)) + testEncode(cb"\ue000")(Unmappable(1)) + testEncode(cb"\uffff")(Unmappable(1)) + testEncode(cb"\ud835\udcd7")(Unmappable(2)) + + testEncode(cb"éA")(Unmappable(1), bb"41") + testEncode(cb"\u0080A")(Unmappable(1), bb"41") + testEncode(cb"\u00ffA")(Unmappable(1), bb"41") + testEncode(cb"\u0100A")(Unmappable(1), bb"41") + testEncode(cb"\u07ffA")(Unmappable(1), bb"41") + testEncode(cb"\ue000A")(Unmappable(1), bb"41") + testEncode(cb"\uffffA")(Unmappable(1), bb"41") + testEncode(cb"\ud835\udcd7A")(Unmappable(2), bb"41") + + // Single UTF-16 surrogates + testEncode(cb"\ud800")(Malformed(1)) + testEncode(cb"\udaff")(Malformed(1)) + testEncode(cb"\udb80")(Malformed(1)) + testEncode(cb"\udbff")(Malformed(1)) + testEncode(cb"\udc00")(Malformed(1)) + testEncode(cb"\udf80")(Malformed(1)) + testEncode(cb"\udfff")(Malformed(1)) + + // High UTF-16 surrogates not followed by low surrogates + testEncode(cb"\ud800A")(Malformed(1), bb"41") + testEncode(cb"\ud800\ud800")(Malformed(1), Malformed(1)) + testEncode(cb"\ud800\ud835\udcd7")(Malformed(1), Unmappable(2)) + testEncode(cb"\udbffA")(Malformed(1), bb"41") + testEncode(cb"\udbff\udb8f")(Malformed(1), Malformed(1)) + testEncode(cb"\udbff\ud835\udcd7")(Malformed(1), Unmappable(2)) + } + + it("isLegalReplacement") { + val encoder = charset.newEncoder + expect(encoder.isLegalReplacement(Array(0x00.toByte))).toBeTruthy + expect(encoder.isLegalReplacement(Array(0x41.toByte))).toBeTruthy + expect(encoder.isLegalReplacement(Array('?'.toByte))).toBeTruthy + expect(encoder.isLegalReplacement(Array(0x80.toByte))).toBeTruthy + expect(encoder.isLegalReplacement(Array(0xff.toByte))).toBeTruthy + } + } +} diff --git a/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/UTF16Test.scala b/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/UTF16Test.scala new file mode 100644 index 0000000..85d4eff --- /dev/null +++ b/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/UTF16Test.scala @@ -0,0 +1,155 @@ +/* __ *\ +** ________ ___ / / ___ __ ____ Scala.js Test Suite ** +** / __/ __// _ | / / / _ | __ / // __/ (c) 2013, LAMP/EPFL ** +** __\ \/ /__/ __ |/ /__/ __ |/_// /_\ \ http://scala-js.org/ ** +** /____/\___/_/ |_/____/_/ | |__/ /____/ ** +** |/____/ ** +\* */ +package scala.scalajs.testsuite.niocharset + +import java.nio._ +import java.nio.charset._ + +import scala.scalajs.niocharset.StandardCharsets + +import BaseCharsetTest._ + +abstract class BaseUTF16Test(charset: Charset) extends BaseCharsetTest(charset) { + describe(charset.name) { + it("decode") { + // ASCII characters + testDecode(bb"0042 006f 006e 006a 006f 0075 0072")(cb"Bonjour") + + // Other characters without surrogate pairs + testDecode(bb"0047 0072 00fc 00df 0020 0047 006f 0074 0074")(cb"Grüß Gott") + testDecode(bb"039a 03b1 03bb 03b7 03bc 03ad 03c1 03b1")(cb"Καλημέρα") + testDecode(bb"0635 0628 0627 062d 0020 0627 0644 062e 064a 0631")(cb"صباح الخير") + testDecode(bb"3053 3093 306b 3061 306f")(cb"こんにちは") + testDecode(bb"0414 043e 0431 0440 044b 0439 0020 0434 0435 043d 044c")(cb"Добрый день") + testDecode(bb"4f60 597d")(cb"你好") + + // 4-byte characters + testDecode(bb"d835 dcd7 d835 dcee d835 dcf5 d835 dcf5 d835 dcf8")( + cb"\ud835\udcd7\ud835\udcee\ud835\udcf5\ud835\udcf5\ud835\udcf8") + + testDecode(bb"")(cb"") + + // Here begin the sequences with at least one error + + // Single UTF-16 surrogates + testDecode(bb"d800")(Malformed(2)) + testDecode(bb"daff")(Malformed(2)) + testDecode(bb"db80")(Malformed(2)) + testDecode(bb"dbff")(Malformed(2)) + testDecode(bb"dc00")(Malformed(2)) + testDecode(bb"df80")(Malformed(2)) + testDecode(bb"dfff")(Malformed(2)) + + // High UTF-16 surrogates not followed by low surrogates + testDecode(bb"d800 0041")(Malformed(2), cb"A") + testDecode(bb"d800 d800")(Malformed(2), Malformed(2)) + testDecode(bb"d800 d835 dcd7")(Malformed(2), cb"\ud835\udcd7") + testDecode(bb"dbff 0041")(Malformed(2), cb"A") + testDecode(bb"dbff db8f")(Malformed(2), Malformed(2)) + testDecode(bb"dbff d835 dcd7")(Malformed(2), cb"\ud835\udcd7") + + // Lonely byte at the end + testDecode(bb"0041 41")(cb"A", Malformed(1)) + } + + it("encode") { + // ASCII characters + testEncode(cb"Bonjour")(bb"0042 006f 006e 006a 006f 0075 0072") + + // Other characters without surrogate pairs + testEncode(cb"Grüß Gott")(bb"0047 0072 00fc 00df 0020 0047 006f 0074 0074") + testEncode(cb"Καλημέρα")(bb"039a 03b1 03bb 03b7 03bc 03ad 03c1 03b1") + testEncode(cb"صباح الخير")(bb"0635 0628 0627 062d 0020 0627 0644 062e 064a 0631") + testEncode(cb"こんにちは")(bb"3053 3093 306b 3061 306f") + testEncode(cb"Добрый день")(bb"0414 043e 0431 0440 044b 0439 0020 0434 0435 043d 044c") + testEncode(cb"你好")(bb"4f60 597d") + + // 4-byte characters + testEncode(cb"\ud835\udcd7\ud835\udcee\ud835\udcf5\ud835\udcf5\ud835\udcf8")( + bb"d835 dcd7 d835 dcee d835 dcf5 d835 dcf5 d835 dcf8") + + testEncode(cb"")(bb"") + + // Here begin the sequences with at least one error + + // Single UTF-16 surrogates + testEncode(cb"\ud800")(Malformed(1)) + testEncode(cb"\udaff")(Malformed(1)) + testEncode(cb"\udb80")(Malformed(1)) + testEncode(cb"\udbff")(Malformed(1)) + testEncode(cb"\udc00")(Malformed(1)) + testEncode(cb"\udf80")(Malformed(1)) + testEncode(cb"\udfff")(Malformed(1)) + + // High UTF-16 surrogates not followed by low surrogates + testEncode(cb"\ud800A")(Malformed(1), bb"0041") + testEncode(cb"\ud800\ud800")(Malformed(1), Malformed(1)) + testEncode(cb"\ud800\ud835\udcd7")(Malformed(1), bb"d835 dcd7") + testEncode(cb"\udbffA")(Malformed(1), bb"0041") + testEncode(cb"\udbff\udb8f")(Malformed(1), Malformed(1)) + testEncode(cb"\udbff\ud835\udcd7")(Malformed(1), bb"d835 dcd7") + } + } +} + +object UTF16BETest extends BaseUTF16Test(StandardCharsets.UTF_16BE) + +object UTF16LETest extends BaseUTF16Test(StandardCharsets.UTF_16LE) { + override protected def testDecode(in: ByteBuffer)( + outParts: OutPart[CharBuffer]*): Unit = { + flipByteBuffer(in) + super.testDecode(in)(outParts: _*) + } + + override protected def testEncode(in: CharBuffer)( + outParts: OutPart[ByteBuffer]*): Unit = { + for (BufferPart(buf) <- outParts) + flipByteBuffer(buf) + super.testEncode(in)(outParts: _*) + } + + /** Flips all pairs of bytes in a byte buffer, except a potential lonely + * last byte. + */ + def flipByteBuffer(buf: ByteBuffer): Unit = { + buf.mark() + while (buf.remaining() >= 2) { + val high = buf.get() + val low = buf.get() + buf.position(buf.position - 2) + buf.put(low) + buf.put(high) + } + buf.reset() + } +} + +object UTF16Test extends BaseUTF16Test(StandardCharsets.UTF_16) { + def BigEndianBOM = ByteBuffer.wrap(Array(0xfe.toByte, 0xff.toByte)) + + override protected def testDecode(in: ByteBuffer)( + outParts: OutPart[CharBuffer]*): Unit = { + // Without BOM, big endian is assumed + super.testDecode(in)(outParts: _*) + + // With BOM, big endian + val inWithBOM = ByteBuffer.allocate(2+in.remaining) + inWithBOM.put(BigEndianBOM).put(in).flip() + super.testDecode(inWithBOM)(outParts: _*) + + // With BOM, little endian + UTF16LETest.flipByteBuffer(inWithBOM) + super.testDecode(inWithBOM)(outParts: _*) + } + + override protected def testEncode(in: CharBuffer)( + outParts: OutPart[ByteBuffer]*): Unit = { + if (in.remaining == 0) super.testEncode(in)(outParts: _*) + else super.testEncode(in)(BufferPart(BigEndianBOM) +: outParts: _*) + } +} diff --git a/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/UTF8Test.scala b/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/UTF8Test.scala new file mode 100644 index 0000000..fbb6a9c --- /dev/null +++ b/test-suite/src/test/scala/scala/scalajs/testsuite/niocharset/UTF8Test.scala @@ -0,0 +1,240 @@ +/* __ *\ +** ________ ___ / / ___ __ ____ Scala.js Test Suite ** +** / __/ __// _ | / / / _ | __ / // __/ (c) 2013, LAMP/EPFL ** +** __\ \/ /__/ __ |/ /__/ __ |/_// /_\ \ http://scala-js.org/ ** +** /____/\___/_/ |_/____/_/ | |__/ /____/ ** +** |/____/ ** +\* */ +package scala.scalajs.testsuite.niocharset + +import java.nio._ +import java.nio.charset._ + +import scala.scalajs.niocharset.StandardCharsets + +import BaseCharsetTest._ + +object UTF8Test extends BaseCharsetTest(StandardCharsets.UTF_8) { + describe("UTF-8") { + it("decode") { + def OutSeq(elems: OutPart[CharBuffer]*): Seq[OutPart[CharBuffer]] = + Seq[OutPart[CharBuffer]](elems: _*) + + // 1-byte characters + testDecode(bb"42 6f 6e 6a 6f 75 72")(cb"Bonjour") + + // 2-byte characters + testDecode(bb"47 72 c3 bc c3 9f 20 47 6f 74 74")(cb"Grüß Gott") + testDecode(bb"ce 9a ce b1 ce bb ce b7 ce bc ce ad cf 81 ce b1")(cb"Καλημέρα") + testDecode(bb"d8 b5 d8 a8 d8 a7 d8 ad 20 d8 a7 d9 84 d8 ae d9 8a d8 b1")(cb"صباح الخير") + + // 3-byte characters + testDecode(bb"e3 81 93 e3 82 93 e3 81 ab e3 81 a1 e3 81 af")(cb"こんにちは") + testDecode(bb"d0 94 d0 be d0 b1 d1 80 d1 8b d0 b9 20 d0 b4 d0 b5 d0 bd d1 8c")(cb"Добрый день") + testDecode(bb"e4 bd a0 e5 a5 bd")(cb"你好") + + // 4-byte characters + testDecode(bb"f0 9d 93 97 f0 9d 93 ae f0 9d 93 b5 f0 9d 93 b5 f0 9d 93 b8")( + cb"\ud835\udcd7\ud835\udcee\ud835\udcf5\ud835\udcf5\ud835\udcf8") + + testDecode(bb"")(cb"") + + // First sequence of a certain length + testDecode(bb"00")(cb"\u0000") + testDecode(bb"c2 80")(cb"\u0080") + testDecode(bb"e0 a0 80")(cb"\u0800") + testDecode(bb"f0 90 80 80")(cb"\ud800\udc00") + + // Last sequence of a certain length + testDecode(bb"7f")(cb"\u007f") + testDecode(bb"df bf")(cb"\u07ff") + testDecode(bb"ef bf bf")(cb"\uffff") + testDecode(bb"f4 8f bf bf")(cb"\udbff\udfff") + + // Other boundary conditions + testDecode(bb"ed 9f bf")(cb"\ud7ff") + testDecode(bb"ee 80 80")(cb"\ue000") + testDecode(bb"ef bf bd")(cb"\ufffd") + + // Here begin the sequences with at least one error + + // Code point too big + testDecode(bb"f4 90 80 80")(Malformed(4)) + testDecode(bb"41 f4 90 80 80 42")(cb"A", Malformed(4), cb"B") + + // Unexpected continuation bytes (each is reported separately) + testDecode(bb"80")(Malformed(1)) + testDecode(bb"bf")(Malformed(1)) + testDecode(bb"80 80")(Malformed(1), Malformed(1)) + testDecode(bb"80 80 80")(Malformed(1), Malformed(1), Malformed(1)) + testDecode(bb"80 80 80 80")(Malformed(1), Malformed(1), Malformed(1), Malformed(1)) + testDecode(bb"80 80 80 80 80")(Malformed(1), Malformed(1), Malformed(1), Malformed(1), Malformed(1)) + testDecode(bb"41 80 80 42 80 43")(cb"A", Malformed(1), Malformed(1), cb"B", Malformed(1), cb"C") + + // Lonely start characters, separated by spaces + /* We add 2 spaces at the end so that the last 4-byte start does not + * swallow the last space as an underflow. */ + testDecode(bb"${(0xc0 to 0xf4).flatMap(c => Seq(c, 32)) ++ Seq[Byte](32, 32)}")( + (0xc0 to 0xf4).flatMap(i => OutSeq(Malformed(1), cb" ")) ++ OutSeq(cb" "): _*) + // Now we let it swallow the last space + testDecode(bb"${Seq[Byte](32) ++ (0xc0 to 0xf4).flatMap(c => Seq(c, 32))}")( + (0xc0 to 0xf4).flatMap(i => OutSeq(cb" ", Malformed(1))): _*) + + // Sequences with some continuation bytes missing + testDecode(bb"c2")(Malformed(1)) + testDecode(bb"e0")(Malformed(1)) + testDecode(bb"e0 a0")(Malformed(2)) + testDecode(bb"f0")(Malformed(1)) + testDecode(bb"f0 90")(Malformed(2)) + testDecode(bb"f0 90 80")(Malformed(3)) + // and all of them concatenated + testDecode(bb"c2 e0 e0 a0 f0 f0 90 f0 90 80")( + Seq(1, 1, 2, 1, 2, 3).map(Malformed(_)): _*) + // and with normal sequences interspersed + testDecode(bb"c2 41 e0 41 e0 a0 41 f0 41 f0 90 41 f0 90 80 41")( + Seq(1, 1, 2, 1, 2, 3).flatMap(l => Seq[OutPart[CharBuffer]](Malformed(l), cb"A")): _*) + + // Impossible bytes + testDecode(bb"fe")(Malformed(1)) + testDecode(bb"ff")(Malformed(1)) + testDecode(bb"fe fe ff ff")(Malformed(1), Malformed(1), Malformed(1), Malformed(1)) + // Old 5-byte and 6-byte starts + testDecode(bb"f8 80 80 80 af")( + Malformed(1), Malformed(1), Malformed(1), Malformed(1), Malformed(1)) + testDecode(bb"fc 80 80 80 80 af")( + Malformed(1), Malformed(1), Malformed(1), Malformed(1), Malformed(1), Malformed(1)) + + // Overlong sequences (encoded with more bytes than necessary) + // Overlong '/' + testDecode(bb"c0 af")(Malformed(2)) + testDecode(bb"e0 80 af")(Malformed(3)) + testDecode(bb"f0 80 80 af")(Malformed(4)) + // Maximum overlong sequences + testDecode(bb"c1 bf")(Malformed(2)) + testDecode(bb"e0 9f bf")(Malformed(3)) + testDecode(bb"f0 8f bf bf")(Malformed(4)) + // Overlong NUL + testDecode(bb"c0 80")(Malformed(2)) + testDecode(bb"e0 80 80")(Malformed(3)) + testDecode(bb"f0 80 80 80")(Malformed(4)) + + // Single UTF-16 surrogates + testDecode(bb"ed a0 80")(Malformed(3)) + testDecode(bb"ed ad bf")(Malformed(3)) + testDecode(bb"ed ae 80")(Malformed(3)) + testDecode(bb"ed af bf")(Malformed(3)) + testDecode(bb"ed b0 80")(Malformed(3)) + testDecode(bb"ed be 80")(Malformed(3)) + testDecode(bb"ed bf bf")(Malformed(3)) + + // Paired UTF-16 surrogates + testDecode(bb"ed a0 80 ed b0 80")(Malformed(3), Malformed(3)) + testDecode(bb"ed a0 80 ed bf bf")(Malformed(3), Malformed(3)) + testDecode(bb"ed ad bf ed b0 80")(Malformed(3), Malformed(3)) + testDecode(bb"ed ad bf ed bf bf")(Malformed(3), Malformed(3)) + testDecode(bb"ed ae 80 ed b0 80")(Malformed(3), Malformed(3)) + testDecode(bb"ed ae 80 ed bf bf")(Malformed(3), Malformed(3)) + testDecode(bb"ed af bf ed b0 80")(Malformed(3), Malformed(3)) + testDecode(bb"ed af bf ed bf bf")(Malformed(3), Malformed(3)) + } + + it("encode") { + def OutSeq(elems: OutPart[ByteBuffer]*): Seq[OutPart[ByteBuffer]] = + Seq[OutPart[ByteBuffer]](elems: _*) + + // 1-byte characters + testEncode(cb"Bonjour")(bb"42 6f 6e 6a 6f 75 72") + + // 2-byte characters + testEncode(cb"Grüß Gott")(bb"47 72 c3 bc c3 9f 20 47 6f 74 74") + testEncode(cb"Καλημέρα")(bb"ce 9a ce b1 ce bb ce b7 ce bc ce ad cf 81 ce b1") + testEncode(cb"صباح الخير")(bb"d8 b5 d8 a8 d8 a7 d8 ad 20 d8 a7 d9 84 d8 ae d9 8a d8 b1") + + // 3-byte characters + testEncode(cb"こんにちは")(bb"e3 81 93 e3 82 93 e3 81 ab e3 81 a1 e3 81 af") + testEncode(cb"Добрый день")(bb"d0 94 d0 be d0 b1 d1 80 d1 8b d0 b9 20 d0 b4 d0 b5 d0 bd d1 8c") + testEncode(cb"你好")(bb"e4 bd a0 e5 a5 bd") + + // 4-byte characters + testEncode(cb"\ud835\udcd7\ud835\udcee\ud835\udcf5\ud835\udcf5\ud835\udcf8")( + bb"f0 9d 93 97 f0 9d 93 ae f0 9d 93 b5 f0 9d 93 b5 f0 9d 93 b8") + + testEncode(cb"")(bb"") + + // First sequence of a certain length + testEncode(cb"\u0000")(bb"00") + testEncode(cb"\u0080")(bb"c2 80") + testEncode(cb"\u0800")(bb"e0 a0 80") + testEncode(cb"\ud800\udc00")(bb"f0 90 80 80") + + // Last sequence of a certain length + testEncode(cb"\u007f")(bb"7f") + testEncode(cb"\u07ff")(bb"df bf") + testEncode(cb"\uffff")(bb"ef bf bf") + testEncode(cb"\udbff\udfff")(bb"f4 8f bf bf") + + // Other boundary conditions + testEncode(cb"\ud7ff")(bb"ed 9f bf") + testEncode(cb"\ue000")(bb"ee 80 80") + testEncode(cb"\ufffd")(bb"ef bf bd") + + // Here begin the sequences with at least one error + + // Single UTF-16 surrogates + testEncode(cb"\ud800")(Malformed(1)) + testEncode(cb"\udaff")(Malformed(1)) + testEncode(cb"\udb80")(Malformed(1)) + testEncode(cb"\udbff")(Malformed(1)) + testEncode(cb"\udc00")(Malformed(1)) + testEncode(cb"\udf80")(Malformed(1)) + testEncode(cb"\udfff")(Malformed(1)) + + // High UTF-16 surrogates not followed by low surrogates + testEncode(cb"\ud800A")(Malformed(1), bb"41") + testEncode(cb"\ud800\ud800")(Malformed(1), Malformed(1)) + testEncode(cb"\ud800\ud835\udcd7")(Malformed(1), bb"f0 9d 93 97") + testEncode(cb"\udbffA")(Malformed(1), bb"41") + testEncode(cb"\udbff\udb8f")(Malformed(1), Malformed(1)) + testEncode(cb"\udbff\ud835\udcd7")(Malformed(1), bb"f0 9d 93 97") + } + + it("isLegalReplacement") { + val encoder = charset.newEncoder + + // The good ones + + expect(encoder.isLegalReplacement(Array(0x00.toByte))).toBeTruthy + expect(encoder.isLegalReplacement(Array(0x41.toByte))).toBeTruthy + expect(encoder.isLegalReplacement(Array('?'.toByte))).toBeTruthy + expect(encoder.isLegalReplacement(Array(0x7f.toByte))).toBeTruthy + + expect(encoder.isLegalReplacement( + Array(0xc2.toByte, 0x80.toByte))).toBeTruthy + expect(encoder.isLegalReplacement( + Array(0xdf.toByte, 0xbf.toByte))).toBeTruthy + + expect(encoder.isLegalReplacement( + Array(0xe0.toByte, 0xa0.toByte, 0x80.toByte))).toBeTruthy + expect(encoder.isLegalReplacement( + Array(0xef.toByte, 0xbf.toByte, 0xbf.toByte))).toBeTruthy + + expect(encoder.isLegalReplacement( + Array(0xf0.toByte, 0x90.toByte, 0x80.toByte, 0x80.toByte))).toBeTruthy + expect(encoder.isLegalReplacement( + Array(0xf4.toByte, 0x8f.toByte, 0xbf.toByte, 0xbf.toByte))).toBeTruthy + + // The bad ones + + expect(encoder.isLegalReplacement(Array(0x80.toByte))).toBeFalsy + expect(encoder.isLegalReplacement(Array(0xbf.toByte))).toBeFalsy + expect(encoder.isLegalReplacement(Array(0xc2.toByte))).toBeFalsy + expect(encoder.isLegalReplacement(Array(0xdf.toByte))).toBeFalsy + expect(encoder.isLegalReplacement(Array(0xe0.toByte))).toBeFalsy + expect(encoder.isLegalReplacement(Array(0xef.toByte))).toBeFalsy + expect(encoder.isLegalReplacement(Array(0xf4.toByte))).toBeFalsy + + expect(encoder.isLegalReplacement( + Array(0xe0.toByte, 0x80.toByte))).toBeFalsy + } + } +} |