diff options
-rw-r--r-- | README.md | 5 | ||||
-rw-r--r-- | core/src/main/scala/com/softwaremill/sttp/UriInterpolator.scala | 284 | ||||
-rw-r--r-- | core/src/main/scala/com/softwaremill/sttp/package.scala | 2 | ||||
-rw-r--r-- | tests/src/test/scala/com/softwaremill/sttp/UriInterpolatorTests.scala | 64 |
4 files changed, 322 insertions, 33 deletions
@@ -24,10 +24,11 @@ println(response.body) // has type String as specified when ## How is sttp different from other libraries? -* immutable request builder which doesn't require the URI to be specified upfront. Allows defining partial requests +* immutable request builder which doesn't impose any order in which request parameters need to be specified. +One consequence of that approach is that the URI to be specified upfront. Allows defining partial requests which contain common cookies/headers/options, which can later be specialized using a specific URI and HTTP method. * support for multiple backends, both synchronous and asynchronous, with backend-specific streaming support -* TODO URI interpolator with optional parameters support +* URI interpolator with optional parameters support ## Usage diff --git a/core/src/main/scala/com/softwaremill/sttp/UriInterpolator.scala b/core/src/main/scala/com/softwaremill/sttp/UriInterpolator.scala index 825985a..b967870 100644 --- a/core/src/main/scala/com/softwaremill/sttp/UriInterpolator.scala +++ b/core/src/main/scala/com/softwaremill/sttp/UriInterpolator.scala @@ -1,33 +1,281 @@ package com.softwaremill.sttp -import java.net.URI +import java.net.{URI, URLEncoder} -// based on https://gist.github.com/teigen/5865923 object UriInterpolator { - private val unreserved = { - val alphanum = (('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9')).toSet - val mark = Set('-', '_', '.', '!', '~', '*', '\'', '(', ')') - alphanum ++ mark - } - - def interpolate(sc: StringContext, args: String*): URI = { + def interpolate(sc: StringContext, args: Any*): URI = { val strings = sc.parts.iterator val expressions = args.iterator - val sb = new StringBuffer(strings.next()) + var ub = UriBuilderStart.parseS(strings.next(), identity) while (strings.hasNext) { - for (c <- expressions.next()) { - if (unreserved(c)) - sb.append(c) + ub = ub.parseE(expressions.next()) + ub = ub.parseS(strings.next(), identity) + } + + new URI(ub.build) + } + + sealed trait UriBuilder { + + /** + * @param doEncode Only values from expressions should be URI-encoded. Strings should be preserved as-is. + */ + def parseS(s: String, doEncode: String => String): UriBuilder + def parseE(e: Any): UriBuilder = e match { + case s: String => parseS(s, encode(_)) + case None => this + case null => this + case Some(x) => parseE(x) + case x => parseS(x.toString, encode(_)) + } + def build: String + } + + val UriBuilderStart = Scheme("") + + case class Scheme(v: String) extends UriBuilder { + + override def parseS(s: String, doEncode: String => String): UriBuilder = { + val splitAtSchemeEnd = s.split("://", 2) + splitAtSchemeEnd match { + case Array(schemeFragment, rest) => + Authority(append(schemeFragment, doEncode)) + .parseS(rest, doEncode) + + case Array(x) => + if (!x.matches("[a-zA-Z0-9+\\.\\-]*")) { + // anything else than the allowed characters in scheme suggest that there is no scheme + // assuming whatever we parsed so far is part of authority, and parsing the rest + // see https://stackoverflow.com/questions/3641722/valid-characters-for-uri-schemes + Authority(Scheme(""), v).parseS(x, doEncode) + } else append(x, doEncode) + } + } + + private def append(x: String, doEncode: String => String): Scheme = + Scheme(v + doEncode(x)) + + override def build: String = if (v.isEmpty) "" else v + "://" + } + + case class Authority(s: Scheme, v: String = "") extends UriBuilder { + + override def parseS(s: String, doEncode: (String) => String): UriBuilder = { + // authority is terminated by /, ?, # or end of string (there might be other /, ?, # later on e.g. in the query) + // see https://tools.ietf.org/html/rfc3986#section-3.2 + s.split("[/\\?#]", 2) match { + case Array(authorityFragment, rest) => + val splitOn = charAfterPrefix(authorityFragment, s) + append(authorityFragment, doEncode).next(splitOn, rest, doEncode) + case Array(x) => append(x, doEncode) + } + } + + private def next(splitOn: Char, + rest: String, + doEncode: (String) => String): UriBuilder = + splitOn match { + case '/' => Path(this).parseS(rest, doEncode) + case '?' => Query(Path(this)).parseS(rest, doEncode) + case '#' => Fragment(Query(Path(this))).parseS(rest, doEncode) + } + + override def parseE(e: Any): UriBuilder = e match { + case s: Seq[_] => + val newAuthority = s.map(_.toString).map(encode(_)).mkString(".") + copy(v = v + newAuthority) + case x => super.parseE(x) + } + + override def build: String = { + // remove dangling "." which might occur due to optional authority fragments + val v2 = if (v.startsWith(".")) v.substring(1) else v + val v3 = if (v.endsWith(".")) v2.substring(0, v2.length - 1) else v2 + + s.build + v3 + } + + private def append(x: String, doEncode: String => String): Authority = + copy(v = v + doEncode(x)) + } + + case class Path(a: Authority, vs: Vector[String] = Vector.empty) + extends UriBuilder { + + override def parseS(s: String, doEncode: (String) => String): UriBuilder = { + // path is terminated by ?, # or end of string (there might be other ?, # later on e.g. in the query) + // see https://tools.ietf.org/html/rfc3986#section-3.3 + s.split("[\\?#]", 2) match { + case Array(pathFragments, rest) => + val splitOn = charAfterPrefix(pathFragments, s) + append(pathFragments, doEncode).next(splitOn, rest, doEncode) + case Array(x) => append(x, doEncode) + } + } + + private def next(splitOn: Char, + rest: String, + doEncode: (String) => String): UriBuilder = + splitOn match { + case '?' => Query(this).parseS(rest, doEncode) + case '#' => Fragment(Query(this)).parseS(rest, doEncode) + } + + override def parseE(e: Any): UriBuilder = e match { + case s: Seq[_] => + val newFragments = s.map(_.toString).map(encode(_)) + copy(vs = vs ++ newFragments) + + case x => super.parseE(x) + } + + override def build: String = { + val v = if (vs.isEmpty) "" else "/" + vs.mkString("/") + a.build + v + } + + private def append(fragments: String, doEncode: String => String): Path = { + copy(vs = vs ++ fragments.split("/").map(doEncode)) + } + } + + type QueryFragment = (Option[String], Option[String]) + + case class Query(p: Path, fs: Vector[QueryFragment] = Vector.empty) + extends UriBuilder { + + override def parseS(s: String, doEncode: (String) => String): UriBuilder = { + s.split("#", 2) match { + case Array(queryFragment, rest) => + Fragment(appendS(queryFragment)).parseS(rest, doEncode) + + case Array(x) => appendS(x) + } + } + + override def parseE(e: Any): UriBuilder = e match { + case m: Map[_, _] => + val newFragments = m.map { + case (k, v) => + (Some(encode(k, query = true)), Some(encode(v, query = true))) + } + copy(fs = fs ++ newFragments) + + case s: Seq[_] => + val newFragments = s.map { + case (k, v) => + (Some(encode(k, query = true)), Some(encode(v, query = true))) + case x => (Some(encode(x, query = true)), None) + } + copy(fs = fs ++ newFragments) + + case s: String => appendE(Some(encode(s, query = true))) + case None => appendE(None) + case null => appendE(None) + case Some(x) => parseE(x) + case x => appendE(Some(encode(x.toString, query = true))) + } + + override def build: String = { + val fragments = fs.flatMap { + case (None, None) => None + case (Some(k), None) => Some(k) + case (k, v) => Some(k.getOrElse("") + "=" + v.getOrElse("")) + } + + val query = if (fragments.isEmpty) "" else "?" + fragments.mkString("&") + + p.build + query + } + + private def appendS(queryFragment: String): Query = { + + val newVs = queryFragment.split("&").map { nv => + /* + - empty -> (None, None) + - k=v -> (Some(k), Some(v)) + - k= -> (Some(k), Some("")) + - k -> (Some(k), None) + - = -> (None, Some("")) + - =v -> (None, Some(v)) + */ + if (nv.isEmpty) (None, None) + else if (nv.startsWith("=")) (None, Some(nv.substring(1))) else - for (b <- c.toString.getBytes("UTF-8")) { - sb.append("%") - sb.append("%02X".format(b)) + nv.split("=", 2) match { + case Array(n, v) => (Some(n), Some(v)) + case Array(n) => (Some(n), None) } } - sb.append(strings.next()) + + // it's possible that the current-last and new-first query fragments + // are indeed two parts of a single fragment. Attempting to merge, + // if possible + val (currentInit, currentLastV) = fs.splitAt(fs.length - 1) + val (newHeadV, newTail) = newVs.splitAt(1) + + val mergedOpt = for { + currentLast <- currentLastV.headOption + newHead <- newHeadV.headOption + } yield { + currentInit ++ merge(currentLast, newHead) ++ newTail + } + + val combinedVs = mergedOpt match { + case None => fs ++ newVs // either current or new fragments are empty + case Some(merged) => merged + } + + copy(fs = combinedVs) + } + + private def merge(last: QueryFragment, + first: QueryFragment): Vector[QueryFragment] = { + /* + Only some combinations of fragments are possible. Part of them is + already handled in `appendE` (specifically, any expressions of + the form k=$v). Here we have to handle: $k=$v and $k=v. + */ + (last, first) match { + case ((Some(k), None), (None, Some(""))) => + Vector((Some(k), Some(""))) // k + = => k= + case ((Some(k), None), (None, Some(v))) => + Vector((Some(k), Some(v))) // k + =v => k=v + case (x, y) => Vector(x, y) + } } - new URI(sb.toString) + + private def appendE(vo: Option[String]): Query = fs.lastOption match { + case Some((Some(k), Some(""))) => + // k= + Some(v) -> k=v; k= + None -> remove parameter + vo match { + case None => copy(fs = fs.init) + case Some(v) => copy(fs = fs.init :+ (Some(k), Some(v))) + } + case _ => copy(fs = fs :+ (vo, None)) + } + } + + case class Fragment(q: Query, v: String = "") extends UriBuilder { + override def parseS(s: String, doEncode: (String) => String): UriBuilder = { + copy(v = v + doEncode(s)) + } + + override def build: String = q.build + (if (v.isEmpty) "" else s"#$v") + } + + private def encode(s: Any, query: Boolean = false): String = { + val encoded = URLEncoder.encode(String.valueOf(s), "UTF-8") + // space is encoded as a +, which is only valid in the query; + // in other contexts, it must be percent-encoded + // see https://stackoverflow.com/questions/2678551/when-to-encode-space-to-plus-or-20 + if (!query) encoded.replaceAll("\\+", "%20") else encoded + } + + private def charAfterPrefix(prefix: String, whole: String): Char = { + val pl = prefix.length + whole.substring(pl, pl + 1).charAt(0) } } diff --git a/core/src/main/scala/com/softwaremill/sttp/package.scala b/core/src/main/scala/com/softwaremill/sttp/package.scala index d757bc0..79d9a17 100644 --- a/core/src/main/scala/com/softwaremill/sttp/package.scala +++ b/core/src/main/scala/com/softwaremill/sttp/package.scala @@ -244,6 +244,6 @@ package object sttp { s"$ct; charset=$enc" implicit class UriContext(val sc: StringContext) extends AnyVal { - def uri(args: String*): URI = UriInterpolator.interpolate(sc, args: _*) + def uri(args: Any*): URI = UriInterpolator.interpolate(sc, args: _*) } } diff --git a/tests/src/test/scala/com/softwaremill/sttp/UriInterpolatorTests.scala b/tests/src/test/scala/com/softwaremill/sttp/UriInterpolatorTests.scala index 0f30f8a..f0eeb1e 100644 --- a/tests/src/test/scala/com/softwaremill/sttp/UriInterpolatorTests.scala +++ b/tests/src/test/scala/com/softwaremill/sttp/UriInterpolatorTests.scala @@ -2,24 +2,64 @@ package com.softwaremill.sttp import java.net.URI -import org.scalatest.{FlatSpec, Matchers} +import org.scalatest.{FunSuite, Matchers} -class UriInterpolatorTests extends FlatSpec with Matchers { +class UriInterpolatorTests extends FunSuite with Matchers { val v1 = "y" val v2 = "a c" - val v2encoded = "a%20c" + val v2queryEncoded = "a+c" + val v2hostEncoded = "a%20c" + val secure = true - val testData: List[(URI, String)] = List( - (uri"http://example.com", "http://example.com"), - (uri"http://example.com?x=y", "http://example.com?x=y"), - (uri"http://example.com?x=$v1", s"http://example.com?x=$v1"), - (uri"http://example.com?x=$v2", s"http://example.com?x=$v2encoded"), - (uri"http://$v1.com", s"http://$v1.com"), - (uri"http://$v1.com?x=$v2", s"http://$v1.com?x=$v2encoded") + val testData: List[(String, List[(URI, String)])] = List( + "basic" -> List( + (uri"http://example.com", "http://example.com"), + (uri"http://example.com?x=y", "http://example.com?x=y") + ), + "scheme" -> List( + (uri"http${if (secure) "s" else ""}://example.com", + s"https://example.com"), + (uri"${if (secure) "https" else "http"}://example.com", + s"https://example.com"), + (uri"${if (secure) "https://" else "http://"}example.com", + s"https://example.com"), + (uri"example.com?a=$v2", s"example.com?a=$v2queryEncoded") + ), + "authority" -> List( + (uri"http://$v1.com", s"http://$v1.com"), + (uri"http://$v2.com", s"http://$v2hostEncoded.com"), + (uri"http://$None.example.com", s"http://example.com"), + (uri"http://${Some("sub")}.example.com", s"http://sub.example.com"), + (uri"http://${List("sub1", "sub2")}.example.com", + s"http://sub1.sub2.example.com"), + (uri"http://${List("sub", "example", "com")}", s"http://sub.example.com") + ), + "authority with parameters" -> List( + (uri"http://$v1.com?x=$v2", s"http://$v1.com?x=$v2queryEncoded") + ), + "query parameter values" -> List( + (uri"http://example.com?x=$v1", s"http://example.com?x=$v1"), + (uri"http://example.com?x=$v2", s"http://example.com?x=$v2queryEncoded") + ), + "optional query parameters" -> List( + (uri"http://example.com?a=$None", s"http://example.com"), + (uri"http://example.com?a=b&c=$None", s"http://example.com?a=b"), + (uri"http://example.com?a=b&c=$None&e=f", s"http://example.com?a=b&e=f"), + (uri"http://example.com?a=${Some(v1)}", s"http://example.com?a=$v1"), + (uri"http://example.com?a=${Some(v1)}&c=d", + s"http://example.com?a=$v1&c=d") + ), + "embed whole url" -> List( + (uri"${"http://example.com/a/b?x=y&1=2"}", + s"http://example.com/a/b?x=y&1=2") + ) ) - for (((interpolated, expected), i) <- testData.zipWithIndex) { - it should s"interpolate to $expected ($i)" in { + for { + (groupName, testCases) <- testData + ((interpolated, expected), i) <- testCases.zipWithIndex + } { + test(s"[$groupName] interpolate to $expected (${i + 1})") { interpolated should be(new URI(expected)) } } |