AcceptEncoding + response decompression

5 files changed, 98 insertions, 13 deletions

diff --git a/README.md b/README.md
index 811da8b..357142e 100644
--- a/README.md
+++ b/README.md
@@ -72,10 +72,10 @@ First, import:
 import com.softwaremill.sttp._
 ```
 
-This brings into scope `sttp`, the empty request, from which all request
-definitions start. This empty request can be customised, each time yielding 
-a new, immutable request description (unless a mutable body is set on the 
-request, such as a byte array).
+This brings into scope `sttp`, the starting request (it's an empty request
+with the `Accept-Encoding: gzip, defalte` header added). This request can 
+be customised, each time yielding a new, immutable request description 
+(unless a mutable body is set on the request, such as a byte array).
 
 For example, we can set a cookie, string-body and specify that this should
 be a `POST` request to a given URI:
diff --git a/akka-http-handler/src/main/scala/com/softwaremill/sttp/akkahttp/AkkaHttpSttpHandler.scala b/akka-http-handler/src/main/scala/com/softwaremill/sttp/akkahttp/AkkaHttpSttpHandler.scala
index fbb00a3..716118c 100644
--- a/akka-http-handler/src/main/scala/com/softwaremill/sttp/akkahttp/AkkaHttpSttpHandler.scala
+++ b/akka-http-handler/src/main/scala/com/softwaremill/sttp/akkahttp/AkkaHttpSttpHandler.scala
@@ -1,10 +1,13 @@
 package com.softwaremill.sttp.akkahttp
 
+import java.io.UnsupportedEncodingException
+
 import akka.actor.{ActorSystem, Terminated}
 import akka.http.scaladsl.Http
+import akka.http.scaladsl.coding.{Deflate, Gzip, NoCoding}
 import akka.http.scaladsl.model.HttpHeader.ParsingResult
 import akka.http.scaladsl.model._
-import akka.http.scaladsl.model.headers.`Content-Type`
+import akka.http.scaladsl.model.headers.{HttpEncodings, `Content-Type`}
 import akka.http.scaladsl.model.ContentTypes.`application/octet-stream`
 import akka.stream.ActorMaterializer
 import akka.stream.scaladsl.{Source, StreamConverters}
@@ -34,7 +37,7 @@ class AkkaHttpSttpHandler(actorSystem: ActorSystem)
       .flatMap(Http().singleRequest(_))
       .flatMap { hr =>
         val code = hr.status.intValue()
-        bodyFromAkka(r.responseAs, hr)
+        bodyFromAkka(r.responseAs, decodeAkkaResponse(hr))
           .map(Response(_, code, headersFromAkka(hr)))
       }
   }
@@ -152,6 +155,19 @@ class AkkaHttpSttpHandler(actorSystem: ActorSystem)
   private def isContentType(header: (String, String)) =
     header._1.toLowerCase.contains(`Content-Type`.lowercaseName)
 
+  // http://doc.akka.io/docs/akka-http/10.0.7/scala/http/common/de-coding.html
+  private def decodeAkkaResponse(response: HttpResponse): HttpResponse = {
+    val decoder = response.encoding match {
+      case HttpEncodings.gzip     => Gzip
+      case HttpEncodings.deflate  => Deflate
+      case HttpEncodings.identity => NoCoding
+      case ce =>
+        throw new UnsupportedEncodingException(s"Unsupported encoding: $ce")
+    }
+
+    decoder.decodeMessage(response)
+  }
+
   def close(): Future[Terminated] = {
     actorSystem.terminate()
   }
diff --git a/core/src/main/scala/com/softwaremill/sttp/HttpURLConnectionSttpHandler.scala b/core/src/main/scala/com/softwaremill/sttp/HttpURLConnectionSttpHandler.scala
index 0a86f97..f5a5971 100644
--- a/core/src/main/scala/com/softwaremill/sttp/HttpURLConnectionSttpHandler.scala
+++ b/core/src/main/scala/com/softwaremill/sttp/HttpURLConnectionSttpHandler.scala
@@ -3,7 +3,9 @@ package com.softwaremill.sttp
 import java.io._
 import java.net.HttpURLConnection
 import java.nio.channels.Channels
+import java.nio.charset.CharacterCodingException
 import java.nio.file.Files
+import java.util.zip.{GZIPInputStream, InflaterInputStream}
 
 import com.softwaremill.sttp.model._
 
@@ -23,6 +25,8 @@ object HttpURLConnectionSttpHandler extends SttpHandler[Id, Nothing] {
       val is = c.getInputStream
       readResponse(c, is, r.responseAs)
     } catch {
+      case e: CharacterCodingException     => throw e
+      case e: UnsupportedEncodingException => throw e
       case _: IOException if c.getResponseCode != -1 =>
         readResponse(c, c.getErrorStream, r.responseAs)
     }
@@ -85,11 +89,15 @@ object HttpURLConnectionSttpHandler extends SttpHandler[Id, Nothing] {
     val headers = c.getHeaderFields.asScala.toVector
       .filter(_._1 != null)
       .flatMap { case (k, vv) => vv.asScala.map((k, _)) }
-    Response(readResponseBody(is, responseAs), c.getResponseCode, headers)
+    val contentEncoding = Option(c.getHeaderField(ContentEncodingHeader))
+    Response(readResponseBody(wrapInput(contentEncoding, is), responseAs),
+             c.getResponseCode,
+             headers)
   }
 
   private def readResponseBody[T](is: InputStream,
                                   responseAs: ResponseAs[T, Nothing]): T = {
+
     def asString(enc: String) = Source.fromInputStream(is, enc).mkString
 
     responseAs match {
@@ -128,4 +136,14 @@ object HttpURLConnectionSttpHandler extends SttpHandler[Id, Nothing] {
         throw new IllegalStateException()
     }
   }
+
+  private def wrapInput(contentEncoding: Option[String],
+                        is: InputStream): InputStream =
+    contentEncoding.map(_.toLowerCase) match {
+      case None            => is
+      case Some("gzip")    => new GZIPInputStream(is)
+      case Some("deflate") => new InflaterInputStream(is)
+      case Some(ce) =>
+        throw new UnsupportedEncodingException(s"Unsupported encoding: $ce")
+    }
 }
diff --git a/core/src/main/scala/com/softwaremill/sttp/package.scala b/core/src/main/scala/com/softwaremill/sttp/package.scala
index d51d6c5..aa2224a 100644
--- a/core/src/main/scala/com/softwaremill/sttp/package.scala
+++ b/core/src/main/scala/com/softwaremill/sttp/package.scala
@@ -181,6 +181,8 @@ package object sttp {
       new SpecifyAuthScheme[U, T, S](AuthorizationHeader, this)
     def proxyAuth: SpecifyAuthScheme[U, T, S] =
       new SpecifyAuthScheme[U, T, S](ProxyAuthorizationHeader, this)
+    def acceptEncoding(encoding: String): RequestT[U, T, S] =
+      header(AcceptEncodingHeader, encoding)
 
     /**
       * Uses the `utf-8` encoding.
@@ -333,11 +335,6 @@ package object sttp {
       rt.header(hn, s"Bearer $token")
   }
 
-  object RequestT {
-    val empty: RequestT[Empty, String, Nothing] =
-      RequestT[Empty, String, Nothing](None, None, NoBody, Vector(), asString)
-  }
-
   type PartialRequest[T, +S] = RequestT[Empty, T, S]
   type Request[T, +S] = RequestT[Id, T, S]
 
@@ -352,6 +349,8 @@ package object sttp {
   private[sttp] val CookieHeader = "Cookie"
   private[sttp] val AuthorizationHeader = "Authorization"
   private[sttp] val ProxyAuthorizationHeader = "Proxy-Authorization"
+  private[sttp] val AcceptEncodingHeader = "Accept-Encoding"
+  private[sttp] val ContentEncodingHeader = "Content-Encoding"
   private val Utf8 = "utf-8"
 
   private val ApplicationOctetStreamContentType = "application/octet-stream"
@@ -359,7 +358,21 @@ package object sttp {
   private val TextPlainContentType = "text/plain"
   private val MultipartFormDataContentType = "multipart/form-data"
 
-  val sttp: RequestT[Empty, String, Nothing] = RequestT.empty
+  /**
+    * An empty request with no headers.
+    */
+  val emptyRequest: RequestT[Empty, String, Nothing] =
+    RequestT[Empty, String, Nothing](None, None, NoBody, Vector(), asString)
+
+  /**
+    * A starting request, with the following modifications comparing to
+    * `emptyRequest`:
+    *
+    * - `Accept-Encoding` set to `gzip, deflate` (handled automatically by the
+    *   library)
+    */
+  val sttp: RequestT[Empty, String, Nothing] =
+    emptyRequest.acceptEncoding("gzip, deflate")
 
   private def contentTypeWithEncoding(ct: String, enc: String) =
     s"$ct; charset=$enc"
diff --git a/tests/src/test/scala/com/softwaremill/sttp/BasicTests.scala b/tests/src/test/scala/com/softwaremill/sttp/BasicTests.scala
index 0f66cc8..96fbbae 100644
--- a/tests/src/test/scala/com/softwaremill/sttp/BasicTests.scala
+++ b/tests/src/test/scala/com/softwaremill/sttp/BasicTests.scala
@@ -4,6 +4,7 @@ import java.io.ByteArrayInputStream
 import java.nio.ByteBuffer
 import java.time.{ZoneId, ZonedDateTime}
 
+import akka.http.scaladsl.coding.{Deflate, Gzip, NoCoding}
 import akka.http.scaladsl.model.{DateTime, FormData}
 import akka.http.scaladsl.model.headers._
 import akka.http.scaladsl.model.headers.CacheDirectives._
@@ -104,6 +105,10 @@ class BasicTests
       }) { userName =>
         complete(s"Hello, $userName!")
       }
+    } ~ path("compress") {
+      encodeResponseWith(Gzip, Deflate, NoCoding) {
+        complete("I'm compressed!")
+      }
     }
 
   override def port = 51823
@@ -133,6 +138,7 @@ class BasicTests
     errorsTests()
     cookiesTests()
     authTests()
+    compressionTests()
 
     def parseResponseTests(): Unit = {
       name should "parse response as string" in {
@@ -338,5 +344,37 @@ class BasicTests
         resp.body should be("Hello, adam!")
       }
     }
+
+    def compressionTests(): Unit = {
+      val compress = sttp.get(uri"$endpoint/compress")
+      val decompressedBody = "I'm compressed!"
+
+      name should "decompress using the default accept encoding header" in {
+        val req = compress
+        val resp = req.send().force()
+        resp.body should be(decompressedBody)
+      }
+
+      name should "decompress using gzip" in {
+        val req =
+          compress.header("Accept-Encoding", "gzip", replaceExisting = true)
+        val resp = req.send().force()
+        resp.body should be(decompressedBody)
+      }
+
+      name should "decompress using deflate" in {
+        val req =
+          compress.header("Accept-Encoding", "deflate", replaceExisting = true)
+        val resp = req.send().force()
+        resp.body should be(decompressedBody)
+      }
+
+      name should "work despite providing an unsupported encoding" in {
+        val req =
+          compress.header("Accept-Encoding", "br", replaceExisting = true)
+        val resp = req.send().force()
+        resp.body should be(decompressedBody)
+      }
+    }
   }
 }