diff options
Diffstat (limited to 'core/src/main/scala')
5 files changed, 289 insertions, 89 deletions
diff --git a/core/src/main/scala/com/softwaremill/sttp/HttpURLConnectionSttpHandler.scala b/core/src/main/scala/com/softwaremill/sttp/HttpURLConnectionSttpHandler.scala index 1ef7fc2..fc9b420 100644 --- a/core/src/main/scala/com/softwaremill/sttp/HttpURLConnectionSttpHandler.scala +++ b/core/src/main/scala/com/softwaremill/sttp/HttpURLConnectionSttpHandler.scala @@ -1,7 +1,7 @@ package com.softwaremill.sttp import java.io._ -import java.net.HttpURLConnection +import java.net.{HttpURLConnection, URL} import java.nio.channels.Channels import java.nio.charset.CharacterCodingException import java.nio.file.Files @@ -15,7 +15,8 @@ import scala.collection.JavaConverters._ object HttpURLConnectionSttpHandler extends SttpHandler[Id, Nothing] { override def send[T](r: Request[T, Nothing]): Response[T] = { - val c = r.uri.toURL.openConnection().asInstanceOf[HttpURLConnection] + val c = + new URL(r.uri.toString).openConnection().asInstanceOf[HttpURLConnection] c.setRequestMethod(r.method.m) r.headers.foreach { case (k, v) => c.setRequestProperty(k, v) } c.setDoInput(true) diff --git a/core/src/main/scala/com/softwaremill/sttp/RequestT.scala b/core/src/main/scala/com/softwaremill/sttp/RequestT.scala index 1aa8770..ff80dd5 100644 --- a/core/src/main/scala/com/softwaremill/sttp/RequestT.scala +++ b/core/src/main/scala/com/softwaremill/sttp/RequestT.scala @@ -1,7 +1,7 @@ package com.softwaremill.sttp import java.io.{File, InputStream} -import java.net.{URI, URLEncoder} +import java.net.URLEncoder import java.nio.ByteBuffer import java.nio.file.Path import java.util.Base64 @@ -25,24 +25,24 @@ import scala.language.higherKinds */ case class RequestT[U[_], T, +S]( method: U[Method], - uri: U[URI], + uri: U[Uri], body: RequestBody[S], headers: Seq[(String, String)], responseAs: ResponseAs[T, S] ) { - def get(uri: URI): Request[T, S] = + def get(uri: Uri): Request[T, S] = this.copy[Id, T, S](uri = uri, method = Method.GET) - def head(uri: URI): Request[T, S] = + def head(uri: Uri): Request[T, S] = this.copy[Id, T, S](uri = uri, method = Method.HEAD) - def post(uri: URI): Request[T, S] = + def post(uri: Uri): Request[T, S] = this.copy[Id, T, S](uri = uri, method = Method.POST) - def put(uri: URI): Request[T, S] = + def put(uri: Uri): Request[T, S] = this.copy[Id, T, S](uri = uri, method = Method.PUT) - def delete(uri: URI): Request[T, S] = + def delete(uri: Uri): Request[T, S] = this.copy[Id, T, S](uri = uri, method = Method.DELETE) - def options(uri: URI): Request[T, S] = + def options(uri: Uri): Request[T, S] = this.copy[Id, T, S](uri = uri, method = Method.OPTIONS) - def patch(uri: URI): Request[T, S] = + def patch(uri: Uri): Request[T, S] = this.copy[Id, T, S](uri = uri, method = Method.PATCH) def contentType(ct: String): RequestT[U, T, S] = @@ -63,7 +63,7 @@ case class RequestT[U[_], T, +S]( this.copy(headers = current :+ (k -> v)) } def headers(hs: Map[String, String]): RequestT[U, T, S] = - this.copy(headers = headers ++ hs.toSeq) + headers(hs.toSeq: _*) def headers(hs: (String, String)*): RequestT[U, T, S] = this.copy(headers = headers ++ hs) def cookie(nv: (String, String)): RequestT[U, T, S] = cookies(nv) diff --git a/core/src/main/scala/com/softwaremill/sttp/Uri.scala b/core/src/main/scala/com/softwaremill/sttp/Uri.scala new file mode 100644 index 0000000..1ee4337 --- /dev/null +++ b/core/src/main/scala/com/softwaremill/sttp/Uri.scala @@ -0,0 +1,184 @@ +package com.softwaremill.sttp + +import java.net.URLEncoder + +import com.softwaremill.sttp.QueryFragment.{KeyValue, Plain} + +import scala.annotation.tailrec +import scala.collection.immutable.Seq + +/** + * @param queryFragments Either key-value pairs, or plain query fragments. + * Key value pairs will be serialized as `k=v`, and blocks of key-value + * pairs will be combined using `&`. Note that no `&` or other separators + * are added around plain query fragments - if required, they need to be + * added manually as part of the plain query fragment. + */ +case class Uri(scheme: String, + host: String, + port: Option[Int], + path: Seq[String], + queryFragments: Seq[QueryFragment], + fragment: Option[String]) { + + def this(host: String) = + this("http", host, None, Vector.empty, Vector.empty, None) + def this(host: String, port: Int) = + this("http", host, Some(port), Vector.empty, Vector.empty, None) + def this(host: String, port: Int, path: Seq[String]) = + this("http", host, Some(port), path, Vector.empty, None) + def this(scheme: String, host: String) = + this(scheme, host, None, Vector.empty, Vector.empty, None) + def this(scheme: String, host: String, port: Int) = + this(scheme, host, Some(port), Vector.empty, Vector.empty, None) + def this(scheme: String, host: String, port: Int, path: Seq[String]) = + this(scheme, host, Some(port), path, Vector.empty, None) + + def scheme(s: String): Uri = this.copy(scheme = s) + + def host(h: String): Uri = this.copy(host = h) + + def port(p: Int): Uri = this.copy(port = Some(p)) + + def port(p: Option[Int]): Uri = this.copy(port = p) + + def path(p: String): Uri = { + // removing the leading slash, as it is added during serialization anyway + val pWithoutLeadingSlash = if (p.startsWith("/")) p.substring(1) else p + val ps = pWithoutLeadingSlash.split("/", -1).toList + this.copy(path = ps) + } + + def path(p1: String, p2: String, ps: String*): Uri = + this.copy(path = p1 :: p2 :: ps.toList) + + def path(ps: scala.collection.Seq[String]): Uri = this.copy(path = ps.toList) + + /** + * Adds the given parameter to the query. + */ + def param(k: String, v: String): Uri = params(k -> v) + + /** + * Adds the given parameters to the query. + */ + def params(ps: Map[String, String]): Uri = params(ps.toSeq: _*) + + /** + * Adds the given parameters to the query. + */ + def params(ps: (String, String)*): Uri = { + this.copy(queryFragments = queryFragments ++ ps.map { + case (k, v) => KeyValue(k, v) + }) + } + + def paramsMap: Map[String, String] = paramsSeq.toMap + + def paramsSeq: Seq[(String, String)] = queryFragments.collect { + case KeyValue(k, v, _, _) => k -> v + } + + def queryFragment(qf: QueryFragment): Uri = + this.copy(queryFragments = queryFragments :+ qf) + + def fragment(f: String): Uri = this.copy(fragment = Some(f)) + + def fragment(f: Option[String]): Uri = this.copy(fragment = f) + + override def toString: String = { + val schemeS = encode(scheme) + val hostS = encode(host) + val portS = port.fold("")(":" + _) + val pathPrefixS = if (path.isEmpty) "" else "/" + val pathS = path.map(encode).mkString("/") + val queryPrefixS = if (queryFragments.isEmpty) "" else "?" + + @tailrec + def encodeQueryFragments(qfs: List[QueryFragment], + previousWasKV: Boolean, + sb: StringBuilder): String = qfs match { + case Nil => sb.toString() + + case Plain(v, re) :: t => + encodeQueryFragments(t, + previousWasKV = false, + sb.append(encodeQuery(v, re))) + + case KeyValue(k, v, reK, reV) :: t => + if (previousWasKV) sb.append("&") + sb.append(encodeQuery(k, reK)).append("=").append(encodeQuery(v, reV)) + encodeQueryFragments(t, previousWasKV = true, sb) + } + + val queryS = encodeQueryFragments(queryFragments.toList, + previousWasKV = false, + new StringBuilder()) + val fragS = fragment.fold("")("#" + _) + s"$schemeS://$hostS$portS$pathPrefixS$pathS$queryPrefixS$queryS$fragS" + } + + private def encode(s: Any): String = { + // space is encoded as a +, which is only valid in the query; + // in other contexts, it must be percent-encoded; see + // https://stackoverflow.com/questions/2678551/when-to-encode-space-to-plus-or-20 + URLEncoder.encode(String.valueOf(s), "UTF-8").replaceAll("\\+", "%20") + } + + private def encodeQuery(s: String, relaxed: Boolean): String = + if (relaxed) encodeQueryRelaxed(s) + else + URLEncoder.encode(String.valueOf(s), "UTF-8") + + private val relaxedQueryAllowedCharacters = { + // https://stackoverflow.com/questions/2322764/what-characters-must-be-escaped-in-an-http-query-string + val alphanum = (('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9')).toSet + val special = Set('/', '?', ':', '@', '-', '.', '_', '~', '!', '$', '&', + '\'', '(', ')', '*', '+', ',', ';', '=') + alphanum ++ special + } + + private def encodeQueryRelaxed(s: String): String = { + val sb = new StringBuilder() + // based on https://gist.github.com/teigen/5865923 + for (c <- s) { + if (relaxedQueryAllowedCharacters(c)) sb.append(c) + else { + for (b <- c.toString.getBytes("UTF-8")) { + sb.append("%") + sb.append("%02X".format(b)) + } + } + } + sb.toString + } +} + +sealed trait QueryFragment +object QueryFragment { + + /** + * @param keyRelaxedEncoding See [[Plain.relaxedEncoding]] + * @param valueRelaxedEncoding See [[Plain.relaxedEncoding]] + */ + case class KeyValue(k: String, + v: String, + keyRelaxedEncoding: Boolean = false, + valueRelaxedEncoding: Boolean = false) + extends QueryFragment + + /** + * A query fragment which will be inserted into the query, without and + * preceding or following separators. Allows constructing query strings + * which are not (only) &-separated key-value pairs. + * + * @param relaxedEncoding Should characters, which are allowed in the query + * string, but normally escaped be left unchanged. These characters are: + * {{{ + * /?:@-._~!$&()*+,;= + * }}} + * See: [[https://stackoverflow.com/questions/2322764/what-characters-must-be-escaped-in-an-http-query-string]] + */ + case class Plain(v: String, relaxedEncoding: Boolean = false) + extends QueryFragment +} diff --git a/core/src/main/scala/com/softwaremill/sttp/UriInterpolator.scala b/core/src/main/scala/com/softwaremill/sttp/UriInterpolator.scala index 3e2ebcf..26b9827 100644 --- a/core/src/main/scala/com/softwaremill/sttp/UriInterpolator.scala +++ b/core/src/main/scala/com/softwaremill/sttp/UriInterpolator.scala @@ -1,10 +1,10 @@ package com.softwaremill.sttp -import java.net.{URI, URLEncoder} +import scala.annotation.tailrec object UriInterpolator { - def interpolate(sc: StringContext, args: Any*): URI = { + def interpolate(sc: StringContext, args: Any*): Uri = { val strings = sc.parts.iterator val expressions = args.iterator var ub = UriBuilderStart.parseS(strings.next()) @@ -14,20 +14,20 @@ object UriInterpolator { ub = ub.parseS(strings.next()) } - new URI(ub.build) + ub.build } sealed trait UriBuilder { def parseS(s: String): UriBuilder def parseE(e: Any): UriBuilder - def build: String + def build: Uri - protected def parseE_asEncodedS_skipNone(e: Any): UriBuilder = e match { - case s: String => parseS(encode(s)) + protected def parseE_skipNone(e: Any): UriBuilder = e match { + case s: String => parseS(s) case None => this case null => this case Some(x) => parseE(x) - case x => parseS(encode(x.toString)) + case x => parseS(x.toString) } } @@ -48,30 +48,16 @@ object UriInterpolator { // there is no scheme assuming whatever we parsed so far is part of // authority, and parsing the rest; see // https://stackoverflow.com/questions/3641722/valid-characters-for-uri-schemes - Authority(Scheme(""), v).parseS(x) + Authority(Scheme("http"), v).parseS(x) } else append(x) } } - override def parseE(e: Any): UriBuilder = { - def encodeIfNotInitialEndpoint(s: String) = { - // special case: when this is the first expression, contains a complete - // schema with :// and nothing is yet parsed not escaping the contents - if (v.isEmpty && s.contains("://")) s else encode(s) - } - - e match { - case s: String => parseS(encodeIfNotInitialEndpoint(s)) - case None => this - case null => this - case Some(x) => parseE(x) - case x => parseS(encodeIfNotInitialEndpoint(x.toString)) - } - } + override def parseE(e: Any): UriBuilder = parseE_skipNone(e) private def append(x: String): Scheme = Scheme(v + x) - override def build: String = if (v.isEmpty) "" else v + "://" + override def build: Uri = Uri(v, "", None, Nil, Nil, None) } case class Authority(s: Scheme, v: String = "") extends UriBuilder { @@ -100,22 +86,27 @@ object UriInterpolator { override def parseE(e: Any): UriBuilder = e match { case s: Seq[_] => - val newAuthority = s.map(_.toString).map(encode(_)).mkString(".") + val newAuthority = s.map(_.toString).mkString(".") copy(v = v + newAuthority) - case x => parseE_asEncodedS_skipNone(x) + case x => parseE_skipNone(x) } - override def build: String = { + override def build: Uri = { var vv = v // remove dangling "." which might occur due to optional authority // fragments while (vv.startsWith(".")) vv = vv.substring(1) - while (vv.endsWith(".")) vv = vv.substring(0, vv.length - 1) - s.build + vv + val builtS = s.build + vv.split(":", 2) match { + case Array(host, port) if port.matches("\\d+") => + builtS.copy(host = host, port = Some(port.toInt)) + case Array(x) => builtS.copy(host = x) + } } - private def append(x: String): Authority = copy(v = v + x) + private def append(x: String): Authority = + copy(v = v + x) } case class Path(a: Authority, fs: Vector[String] = Vector.empty) @@ -141,20 +132,16 @@ object UriInterpolator { override def parseE(e: Any): UriBuilder = e match { case s: Seq[_] => - val newFragments = s.map(_.toString).map(encode(_)).map(Some(_)) + val newFragments = s.map(_.toString).map(Some(_)) newFragments.foldLeft(this)(_.appendE(_)) - case s: String => appendE(Some(encode(s))) + case s: String => appendE(Some(s)) case None => appendE(None) case null => appendE(None) case Some(x) => parseE(x) - case x => appendE(Some(encode(x.toString))) + case x => appendE(Some(x.toString)) } - override def build: String = { - // if there is a trailing /, the last path fragment will be empty - val v = if (fs.isEmpty) "" else "/" + fs.mkString("/") - a.build + v - } + override def build: Uri = a.build.copy(path = fs) private def appendS(fragments: String): Path = { if (fragments.isEmpty) this @@ -200,16 +187,16 @@ object UriInterpolator { } override def parseE(e: Any): UriBuilder = e match { - case m: Map[_, _] => parseSeq(m.toSeq) - case s: Seq[_] => parseSeq(s) - case s: String => appendE(Some(encodeQuery(s))) + case m: Map[_, _] => parseSeqE(m.toSeq) + case s: Seq[_] => parseSeqE(s) + case s: String => appendE(Some(s)) case None => appendE(None) case null => appendE(None) case Some(x) => parseE(x) - case x => appendE(Some(encodeQuery(x.toString))) + case x => appendE(Some(x.toString)) } - private def parseSeq(s: Seq[_]): UriBuilder = { + private def parseSeqE(s: Seq[_]): UriBuilder = { val flattenedS = s.flatMap { case (_, None) => None case (k, Some(v)) => Some((k, v)) @@ -219,32 +206,67 @@ object UriInterpolator { } val newFragments = flattenedS.map { case ("", "") => Eq - case (k, "") => K_Eq(encodeQuery(k)) - case ("", v) => Eq_V(encodeQuery(v)) - case (k, v) => K_Eq_V(encodeQuery(k), encodeQuery(v)) - case x => K(encodeQuery(x)) + case (k, "") => K_Eq(k.toString) + case ("", v) => Eq_V(v.toString) + case (k, v) => K_Eq_V(k.toString, v.toString) + case x => K(x.toString) } copy(fs = fs ++ newFragments) } - override def build: String = { - val fragments = fs.flatMap { + override def build: Uri = { + import com.softwaremill.sttp.{QueryFragment => QF} + + val plainSeparator = QF.Plain("&", relaxedEncoding = true) + var fragments: Vector[QF] = fs.flatMap { case Empty => None - case K_Eq_V(k, v) => Some(s"$k=$v") - case K_Eq(k) => Some(s"$k=") - case K(k) => Some(s"$k") - case Eq => Some("=") - case Eq_V(v) => Some(s"=$v") + case K_Eq_V(k, v) => Some(QF.KeyValue(k, v)) + case K_Eq(k) => Some(QF.KeyValue(k, "")) + case K(k) => + // if we have a key-only entry, we encode it as a plain query + // fragment + Some(QF.Plain(k)) + case Eq => Some(QF.KeyValue("", "")) + case Eq_V(v) => Some(QF.KeyValue("", v)) } - val query = if (fragments.isEmpty) "" else "?" + fragments.mkString("&") + // when serialized, plain query fragments don't have & separators + // prepended/appended - hence, if we have parsed them here, they + // need to be added by hand. Adding an & separator between each pair + // of fragments where one of them is plain. For example: + // KV P P KV KV P KV + // becomes: + // KV S P S P S KV KV S P S KV + @tailrec + def addPlainSeparators(qfs: Vector[QF], + previousWasPlain: Boolean, + acc: Vector[QF], + isFirst: Boolean = false): Vector[QF] = qfs match { + case Vector() => acc + case (p: QF.Plain) +: tail if !isFirst => + addPlainSeparators(tail, + previousWasPlain = true, + acc :+ plainSeparator :+ p) + case (p: QF.Plain) +: tail => + addPlainSeparators(tail, previousWasPlain = true, acc :+ p) + case (kv: QF.KeyValue) +: tail if previousWasPlain => + addPlainSeparators(tail, + previousWasPlain = false, + acc :+ plainSeparator :+ kv) + case (kv: QF.KeyValue) +: tail => + addPlainSeparators(tail, previousWasPlain = false, acc :+ kv) + } + + fragments = addPlainSeparators(fragments, + previousWasPlain = false, + Vector(), + isFirst = true) - p.build + query + p.build.copy(queryFragments = fragments) } private def appendS(queryFragment: String): Query = { - - val newVs = queryFragment.split("&").map { nv => + val newVs = queryFragment.split("&", -1).map { nv => if (nv.isEmpty) Empty else if (nv == "=") Eq else if (nv.startsWith("=")) Eq_V(nv.substring(1)) @@ -285,9 +307,10 @@ object UriInterpolator { the form k=$v). Here we have to handle: $k=$v and $k=v. */ (last, first) match { - case (K(k), Eq) => Vector(K_Eq(k)) // k + = => k= - case (K(k), Eq_V(v)) => Vector(K_Eq_V(k, v)) // k + =v => k=v - case (x, y) => Vector(x, y) + case (K(k), Eq) => Vector(K_Eq(k)) // k + = => k= + case (K(k), Eq_V(v)) => + Vector(K_Eq_V(k, v)) // k + =v => k=v + case (x, y) => Vector(x, y) } } @@ -300,29 +323,22 @@ object UriInterpolator { case Some("") => this case Some(v) => copy(fs = fs.init :+ K_Eq_V(k, v)) } - case _ => copy(fs = fs :+ vo.fold[QueryFragment](Empty)(K)) + case _ => + copy(fs = fs :+ vo.fold[QueryFragment](Empty)(K)) } } } case class Fragment(q: Query, v: String = "") extends UriBuilder { - override def parseS(s: String): UriBuilder = copy(v = v + s) + override def parseS(s: String): UriBuilder = + copy(v = v + s) - override def parseE(e: Any): UriBuilder = parseE_asEncodedS_skipNone(e) + override def parseE(e: Any): UriBuilder = parseE_skipNone(e) - override def build: String = q.build + (if (v.isEmpty) "" else s"#$v") + override def build: Uri = + q.build.copy(fragment = if (v.isEmpty) None else Some(v)) } - private def encode(s: Any): String = { - // space is encoded as a +, which is only valid in the query; - // in other contexts, it must be percent-encoded; see - // https://stackoverflow.com/questions/2678551/when-to-encode-space-to-plus-or-20 - URLEncoder.encode(String.valueOf(s), "UTF-8").replaceAll("\\+", "%20") - } - - private def encodeQuery(s: Any): String = - URLEncoder.encode(String.valueOf(s), "UTF-8") - private def charAfterPrefix(prefix: String, whole: String): Char = { val pl = prefix.length whole.substring(pl, pl + 1).charAt(0) diff --git a/core/src/main/scala/com/softwaremill/sttp/package.scala b/core/src/main/scala/com/softwaremill/sttp/package.scala index 5e7b13c..c786720 100644 --- a/core/src/main/scala/com/softwaremill/sttp/package.scala +++ b/core/src/main/scala/com/softwaremill/sttp/package.scala @@ -1,7 +1,6 @@ package com.softwaremill import java.io.{File, InputStream} -import java.net.URI import java.nio.ByteBuffer import java.nio.file.Path @@ -163,6 +162,6 @@ package object sttp { // uri interpolator implicit class UriContext(val sc: StringContext) extends AnyVal { - def uri(args: Any*): URI = UriInterpolator.interpolate(sc, args: _*) + def uri(args: Any*): Uri = UriInterpolator.interpolate(sc, args: _*) } } |