#53: using proper encoding when reading the response body

2 files changed, 27 insertions, 8 deletions

diff --git a/core/src/main/scala/com/softwaremill/sttp/HttpURLConnectionBackend.scala b/core/src/main/scala/com/softwaremill/sttp/HttpURLConnectionBackend.scala
index 083b1c3..65b1a0c 100644
--- a/core/src/main/scala/com/softwaremill/sttp/HttpURLConnectionBackend.scala
+++ b/core/src/main/scala/com/softwaremill/sttp/HttpURLConnectionBackend.scala
@@ -214,24 +214,30 @@ class HttpURLConnectionBackend private (
       .filter(_._1 != null)
       .flatMap { case (k, vv) => vv.asScala.map((k, _)) }
     val contentEncoding = Option(c.getHeaderField(ContentEncodingHeader))
+
+    val charsetFromHeaders = Option(c.getHeaderField(ContentTypeHeader))
+      .flatMap(encodingFromContentType)
+
     val code = c.getResponseCode
     val wrappedIs = wrapInput(contentEncoding, handleNullInput(is))
     val body = if (codeIsSuccess(code)) {
-      Right(readResponseBody(wrappedIs, responseAs))
+      Right(readResponseBody(wrappedIs, responseAs, charsetFromHeaders))
     } else {
-      Left(readResponseBody(wrappedIs, asString))
+      Left(readResponseBody(wrappedIs, asString, charsetFromHeaders))
     }
 
     Response(body, code, headers, Nil)
   }
 
   private def readResponseBody[T](is: InputStream,
-                                  responseAs: ResponseAs[T, Nothing]): T = {
+                                  responseAs: ResponseAs[T, Nothing],
+                                  charset: Option[String]): T = {
 
-    def asString(enc: String) = Source.fromInputStream(is, enc).mkString
+    def asString(enc: String) =
+      Source.fromInputStream(is, charset.getOrElse(enc)).mkString
 
     responseAs match {
-      case MappedResponseAs(raw, g) => g(readResponseBody(is, raw))
+      case MappedResponseAs(raw, g) => g(readResponseBody(is, raw, charset))
 
       case IgnoreResponse =>
         @tailrec def consume(): Unit = if (is.read() != -1) consume()
diff --git a/core/src/main/scala/com/softwaremill/sttp/package.scala b/core/src/main/scala/com/softwaremill/sttp/package.scala
index b641a77..f47e113 100644
--- a/core/src/main/scala/com/softwaremill/sttp/package.scala
+++ b/core/src/main/scala/com/softwaremill/sttp/package.scala
@@ -83,19 +83,27 @@ package object sttp {
   def ignore: ResponseAs[Unit, Nothing] = IgnoreResponse
 
   /**
-    * Uses `utf-8` encoding.
+    * Use the `utf-8` encoding by default, unless specified otherwise in the response headers.
     */
   def asString: ResponseAs[String, Nothing] = asString(Utf8)
+
+  /**
+    * Use the given encoding by default, unless specified otherwise in the response headers.
+    */
   def asString(encoding: String): ResponseAs[String, Nothing] =
     ResponseAsString(encoding)
   def asByteArray: ResponseAs[Array[Byte], Nothing] =
     ResponseAsByteArray
 
   /**
-    * Uses `utf-8` encoding.
+    * Use the `utf-8` encoding by default, unless specified otherwise in the response headers.
     */
   def asParams: ResponseAs[Seq[(String, String)], Nothing] =
     asParams(Utf8)
+
+  /**
+    * Use the given encoding by default, unless specified otherwise in the response headers.
+    */
   def asParams(encoding: String): ResponseAs[Seq[(String, String)], Nothing] =
     asString(encoding).map(ResponseAs.parseParams(_, encoding))
 
@@ -238,9 +246,14 @@ package object sttp {
 
   // util
 
-  private[sttp] def contentTypeWithEncoding(ct: String, enc: String) =
+  private[sttp] def contentTypeWithEncoding(ct: String, enc: String): String =
     s"$ct; charset=$enc"
 
+  private[sttp] def encodingFromContentType(ct: String): Option[String] =
+    ct.split(";").map(_.trim.toLowerCase).collectFirst {
+      case s if s.startsWith("charset=") => s.substring(8)
+    }
+
   private[sttp] def transfer(is: InputStream, os: OutputStream) {
     var read = 0
     val buf = new Array[Byte](1024)