diff options
author | Li Haoyi <haoyi.sg@gmail.com> | 2017-12-14 21:48:32 -0800 |
---|---|---|
committer | Li Haoyi <haoyi.sg@gmail.com> | 2017-12-14 22:11:54 -0800 |
commit | 3a40842d7b3aeedddb9ab5e8261dd48ea4e024b6 (patch) | |
tree | ed0039fa03048f90e9f1ef41ebb1f5b4917bdb52 | |
parent | d38196041663959ed71881d6d83a27d4639f3134 (diff) | |
download | mill-3a40842d7b3aeedddb9ab5e8261dd48ea4e024b6.tar.gz mill-3a40842d7b3aeedddb9ab5e8261dd48ea4e024b6.tar.bz2 mill-3a40842d7b3aeedddb9ab5e8261dd48ea4e024b6.zip |
First pass adding a Jawn build to the test suite. Jawn's own test suite doesn't run properly for some reason
56 files changed, 4529 insertions, 0 deletions
diff --git a/scalaplugin/src/test/resource/jawn/.gitignore b/scalaplugin/src/test/resource/jawn/.gitignore new file mode 100644 index 00000000..2db3b8c0 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/.gitignore @@ -0,0 +1,20 @@ +lib_managed +project7/boot +project7/plugins/src_managed +project7/plugins/project +project/boot +project/build/target +project/plugins/lib_managed +project/plugins/project +project/plugins/src_managed +project/plugins/target +target +.ensime +.ensime_lucene +TAGS +\#*# +*~ +.#* +.lib +.ensime_cache +.idea diff --git a/scalaplugin/src/test/resource/jawn/.travis.yml b/scalaplugin/src/test/resource/jawn/.travis.yml new file mode 100644 index 00000000..5f9f5fe4 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/.travis.yml @@ -0,0 +1,6 @@ +language: scala +sudo: false +jdk: + - oraclejdk8 +script: + - sbt "so test" diff --git a/scalaplugin/src/test/resource/jawn/README.md b/scalaplugin/src/test/resource/jawn/README.md new file mode 100644 index 00000000..6ea33b92 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/README.md @@ -0,0 +1,427 @@ +## Jawn + +"Jawn is for parsing jay-sawn." + +### Origin + +The term "jawn" comes from the Philadelphia area. It conveys about as +much information as "thing" does. I chose the name because I had moved +to Montreal so I was remembering Philly fondly. Also, there isn't a +better way to describe objects encoded in JSON than "things". Finally, +we get a catchy slogan. + +Jawn was designed to parse JSON into an AST as quickly as possible. + +[![Build Status](https://api.travis-ci.org/non/jawn.svg)](https://travis-ci.org/non/jawn) +[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/non/jawn?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Latest version](https://index.scala-lang.org/non/jawn/jawn-parser/latest.svg?color=orange)](https://index.scala-lang.org/non/jawn/jawn-parser) + +### Overview + +Jawn consists of four parts: + +1. A fast, generic JSON parser (`jawn-parser`) +2. A small, somewhat anemic AST (`jawn-ast`) +3. Support packages which parse to third-party ASTs +4. A few helpful utilities (`jawn-util`) + +Currently Jawn is competitive with the fastest Java JSON libraries +(GSON and Jackson) and in the author's benchmarks it often wins. It +seems to be faster than any other Scala parser that exists (as of July +2014). + +Given the plethora of really nice JSON libraries for Scala, the +expectation is that you're probably here for `jawn-parser` or a +support package. + +### Quick Start + +Jawn supports Scala 2.10, 2.11, and 2.12. + +Here's a `build.sbt` snippet that shows you how to depend on Jawn in +your own SBT project: + +```scala +resolvers += Resolver.sonatypeRepo("releases") + +// use this if you just want jawn's parser, and will implement your own facade +libraryDependencies += "org.spire-math" %% "jawn-parser" % "0.11.0" + +// use this if you want jawn's parser and also jawn's ast +libraryDependencies += "org.spire-math" %% "jawn-ast" % "0.11.0" +``` + +If you want to use Jawn's parser with another project's AST, see the +"Supporting external ASTs with Jawn" section. For example, with Spray +you would say: + +```scala +libraryDependencies += "org.spire-math" %% "jawn-spray" % "0.11.0" +``` + +There are a few reasons you might want to do this: + + * The library's built-in parser is significantly slower than Jawn's. + * Jawn supports more input types (`ByteBuffer`, `File`, etc.). + * You need asynchronous JSON parsing. + +(NOTE: previous to version 0.8.3 the support libraries would have been +named `"spray-support"` instead of `"jawn-spray"`.) + +### Dependencies + +*jawn-parser* has no dependencies other than Scala. + +*jawn-ast* depends on *jawn-parser* but nothing else. + +The various support projects (e.g. *jawn-argonaut*) depend on +the library they are supporting. + +### Parsing + +Jawn's parser is both fast and relatively featureful. Assuming you +want to get back an AST of type `J` and you have a `Facade[J]` +defined, you can use the following `parse` signatures: + +```scala +Parser.parseUnsafe[J](String) → J +Parser.parseFromString[J](String) → Try[J] +Parser.parsefromPath[J](String) → Try[J] +Parser.parseFromFile[J](File) → Try[J] +Parser.parseFromChannel[J](ReadableByteChannel) → Try[J] +Parser.parseFromByteBuffer[J](ByteBuffer) → Try[J] +``` + +Jawn also supports asynchronous parsing, which allows users to feed +the parser with data as it is available. There are three modes: + +* `SingleValue` waits to return a single `J` value once parsing is done. +* `UnwrapArray` if the top-level element is an array, return values as they become available. +* `ValueStream` parse one-or-more json values separated by whitespace. + +Here's an example: + +```scala +import jawn.ast +import jawn.AsyncParser +import jawn.ParseException + +val p = ast.JParser.async(mode = AsyncParser.UnwrapArray) + +def chunks: Stream[String] = ??? +def sink(j: ast.JValue): Unit = ??? + +def loop(st: Stream[String]): Either[ParseException, Unit] = + st match { + case s #:: tail => + p.absorb(s) match { + case Right(js) => + js.foreach(sink) + loop(tail) + case Left(e) => + Left(e) + } + case _ => + p.finish().right.map(_.foreach(sink)) + } + +loop(chunks) +``` + +You can also call `jawn.Parser.async[J]` to use async parsing with an +arbitrary data type (provided you also have an implicit `Facade[J]`). + +### Supporting external ASTs with Jawn + +Jawn currently supports six external ASTs directly: + +| AST | 2.10 | 2.11 | 2.12 | +|-----------|--------|--------|-------| +| Argonaut | 6.2 | 6.2 | 6.2 | +| Json4s | 3.5.2 | 3.5.2 | 3.5.2 | +| Play-json | 2.4.11 | 2.5.15 | 2.6.0 | +| Rojoma | 2.4.3 | 2.4.3 | 2.4.3 | +| Rojoma-v3 | 3.7.2 | 3.7.2 | 3.7.2 | +| Spray | 1.3.3 | 1.3.3 | 1.3.3 | + +Each of these subprojects provides a `Parser` object (an instance of +`SupportParser[J]`) that is parameterized on the given project's +AST (`J`). The following methods are available: + +```scala +Parser.parseUnsafe(String) → J +Parser.parseFromString(String) → Try[J] +Parser.parsefromPath(String) → Try[J] +Parser.parseFromFile(File) → Try[J] +Parser.parseFromChannel(ReadableByteChannel) → Try[J] +Parser.parseFromByteBuffer(ByteBuffer) → Try[J] +``` + +These methods parallel those provided by `jawn.Parser`. + +For the following snippets, `XYZ` is one of (`argonaut`, `json4s`, +`play`, `rojoma`, `rojoma-v3` or `spray`): + +This is how you would include the subproject in build.sbt: + +```scala +resolvers += Resolver.sonatypeRepo("releases") + +libraryDependencies += "org.spire-math" %% jawn-"XYZ" % "0.11.0" +``` + +This is an example of how you might use the parser into your code: + +```scala +import jawn.support.XYZ.Parser + +val myResult = Parser.parseFromString(myString) +``` + +### Do-It-Yourself Parsing + +Jawn supports building any JSON AST you need via type classes. You +benefit from Jawn's fast parser while still using your favorite Scala +JSON library. This mechanism is also what allows Jawn to provide +"support" for other libraries' ASTs. + +To include Jawn's parser in your project, add the following +snippet to your `build.sbt` file: + +```scala +resolvers += Resolver.sonatypeRepo("releases") + +libraryDependencies += "org.spire-math" %% "jawn-parser" % "0.11.0" +``` + +To support your AST of choice, you'll want to define a `Facade[J]` +instance, where the `J` type parameter represents the base of your JSON +AST. For example, here's a facade that supports Spray: + +```scala +import spray.json._ +object Spray extends SimpleFacade[JsValue] { + def jnull() = JsNull + def jfalse() = JsFalse + def jtrue() = JsTrue + def jnum(s: String) = JsNumber(s) + def jint(s: String) = JsNumber(s) + def jstring(s: String) = JsString(s) + def jarray(vs: List[JsValue]) = JsArray(vs) + def jobject(vs: Map[String, JsValue]) = JsObject(vs) +} +``` + +Most ASTs will be easy to define using the `SimpleFacade` or +`MutableFacade` traits. However, if an ASTs object or array instances +do more than just wrap a Scala collection, it may be necessary to +extend `Facade` directly. + +You can also look at the facades used by the support projects to help +you create your own. This could also be useful if you wanted to +use an older version of a supported library. + +### Using the AST + +#### Access + +For accessing atomic values, `JValue` supports two sets of +methods: *get-style* methods and *as-style* methods. + +The *get-style* methods return `Some(_)` when called on a compatible +JSON value (e.g. strings can return `Some[String]`, numbers can return +`Some[Double]`, etc.), and `None` otherwise: + +```scala +getBoolean → Option[Boolean] +getString → Option[String] +getLong → Option[Long] +getDouble → Option[Double] +getBigInt → Option[BigInt] +getBigDecimal → Option[BigDecimal] +``` + +In constrast, the *as-style* methods will either return an unwrapped +value (instead of returning `Some(_)`) or throw an exception (instead +of returning `None`): + +```scala +asBoolean → Boolean // or exception +asString → String // or exception +asLong → Long // or exception +asDouble → Double // or exception +asBigInt → BigInt // or exception +asBigDecimal → BigDecimal // or exception +``` + +To access elements of an array, call `get` with an `Int` position: + +```scala +get(i: Int) → JValue // returns JNull if index is illegal +``` + +To access elements of an object, call `get` with a `String` key: + +```scala +get(k: String) → JValue // returns JNull if key is not found +``` + +Both of these methods also return `JNull` if the value is not the +appropraite container. This allows the caller to chain lookups without +having to check that each level is correct: + +```scala +val v: JValue = ??? + +// returns JNull if a problem is encountered in structure of 'v'. +val t: JValue = v.get("novels").get(0).get("title") + +// if 'v' had the right structure and 't' is JString(s), then Some(s). +// otherwise, None. +val titleOrNone: Option[String] = t.getString + +// equivalent to titleOrNone.getOrElse(throw ...) +val titleOrDie: String = t.asString +``` + +#### Updating + +The atomic values (`JNum`, `JBoolean`, `JNum`, and `JString`) are +immutable. + +Objects are fully-mutable and can have items added, removed, or +changed: + +```scala +set(k: String, v: JValue) → Unit +remove(k: String) → Option[JValue] +``` + +If `set` is called on a non-object, an exception will be thrown. +If `remove` is called on a non-object, `None` will be returned. + +Arrays are semi-mutable. Their values can be changed, but their size +is fixed: + +```scala +set(i: Int, v: JValue) → Unit +``` + +If `set` is called on a non-array, or called with an illegal index, an +exception will be thrown. + +(A future version of Jawn may provide an array whose length can be +changed.) + +### Profiling + +Jawn uses [JMH](http://openjdk.java.net/projects/code-tools/jmh/) +along with the [sbt-jmh](https://github.com/ktoso/sbt-jmh) plugin. + +#### Running Benchmarks + +The benchmarks are located in the `benchmark` project. You can run the +benchmarks by typing `benchmark/run` from SBT. There are many +supported arguments, so here are a few examples: + +Run all benchmarks, with 10 warmups, 10 iterations, using 3 threads: + +`benchmark/run -wi 10 -i 10 -f1 -t3` + +Run just the `CountriesBench` test (5 warmups, 5 iterations, 1 thread): + +`benchmark/run -wi 5 -i 5 -f1 -t1 .*CountriesBench` + +#### Benchmark Issues + +Currently, the benchmarks are a bit fiddily. The most obvious symptom +is that if you compile the benchmarks, make changes, and compile +again, you may see errors like: + +``` +[error] (benchmark/jmh:generateJavaSources) java.lang.NoClassDefFoundError: jawn/benchmark/Bla25Bench +``` + +The fix here is to run `benchmark/clean` and try again. + +You will also see intermittent problems like: + +``` +[error] (benchmark/jmh:compile) java.lang.reflect.MalformedParameterizedTypeException +``` + +The solution here is easier (though frustrating): just try it +again. If you continue to have problems, consider cleaning the project +and trying again. + +(In the future I hope to make the benchmarking here a bit more +resilient. Suggestions and pull requests gladly welcome!) + +#### Files + +The benchmarks use files located in `benchmark/src/main/resources`. If +you want to test your own files (e.g. `mydata.json`), you would: + + * Copy the file to `benchmark/src/main/resources/mydata.json`. + * Add the following code to `JmhBenchmarks.scala`: + +```scala +class MyDataBench extends JmhBenchmarks("mydata.json") +``` + +Jawn has been tested with much larger files, e.g. 100M - 1G, but these +are obviously too large to ship with the project. + +With large files, it's usually easier to comment out most of the +benchmarking methods and only test one (or a few) methods. Some of the +slower JSON parsers get *much* slower for large files. + +#### Interpreting the results + +Remember that the benchmarking results you see will vary based on: + + * Hardware + * Java version + * JSON file size + * JSON file structure + * JSON data values + +I have tried to use each library in the most idiomatic and fastest way +possible (to parse the JSON into a simple AST). Pull requests to +update library versions and improve usage are very welcome. + +### Future Work + +More support libraries could be added. + +It's likely that some of Jawn's I/O could be optimized a bit more, and +also made more configurable. The heuristics around all-at-once loading +versus input chunking could definitely be improved. + +In cases where the user doesn't need fast lookups into JSON objects, +an even lighter AST could be used to improve parsing and rendering +speeds. + +Strategies to cache/intern field names of objects could pay big +dividends in some cases (this might require AST changes). + +If you have ideas for any of these (or other ideas) please feel free +to open an issue or pull request so we can talk about it. + +### Disclaimers + +Jawn only supports UTF-8 when parsing bytes. This might change in the +future, but for now that's the target case. You can always decode your +data to a string, and handle the character set decoding using Java's +standard tools. + +Jawn's AST is intended to be very lightweight and simple. It supports +simple access, and limited mutable updates. It intentionally lacks the +power and sophistication of many other JSON libraries. + +### Copyright and License + +All code is available to you under the MIT license, available at +http://opensource.org/licenses/mit-license.php. + +Copyright Erik Osheim, 2012-2017. diff --git a/scalaplugin/src/test/resource/jawn/ast/src/main/scala/jawn/ast/JParser.scala b/scalaplugin/src/test/resource/jawn/ast/src/main/scala/jawn/ast/JParser.scala new file mode 100644 index 00000000..704557cc --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/ast/src/main/scala/jawn/ast/JParser.scala @@ -0,0 +1,35 @@ +package jawn +package ast + +import java.io.File +import java.nio.ByteBuffer +import java.nio.channels.ReadableByteChannel +import scala.util.Try + +object JParser { + implicit val facade = JawnFacade + + def parseUnsafe(s: String): JValue = + new StringParser(s).parse() + + def parseFromString(s: String): Try[JValue] = + Try(new StringParser[JValue](s).parse) + + def parseFromCharSequence(cs: CharSequence): Try[JValue] = + Try(new CharSequenceParser[JValue](cs).parse) + + def parseFromPath(path: String): Try[JValue] = + parseFromFile(new File(path)) + + def parseFromFile(file: File): Try[JValue] = + Try(ChannelParser.fromFile[JValue](file).parse) + + def parseFromChannel(ch: ReadableByteChannel): Try[JValue] = + Try(ChannelParser.fromChannel(ch).parse) + + def parseFromByteBuffer(buf: ByteBuffer): Try[JValue] = + Try(new ByteBufferParser[JValue](buf).parse) + + def async(mode: AsyncParser.Mode): AsyncParser[JValue] = + AsyncParser(mode) +} diff --git a/scalaplugin/src/test/resource/jawn/ast/src/main/scala/jawn/ast/JValue.scala b/scalaplugin/src/test/resource/jawn/ast/src/main/scala/jawn/ast/JValue.scala new file mode 100644 index 00000000..d09347bc --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/ast/src/main/scala/jawn/ast/JValue.scala @@ -0,0 +1,314 @@ +package jawn +package ast + +import java.lang.Double.{isNaN, isInfinite} +import scala.collection.mutable +import scala.util.hashing.MurmurHash3 + +class WrongValueException(e: String, g: String) extends Exception(s"expected $e, got $g") + +class InvalidNumException(s: String) extends Exception(s"invalid number: $s") + +sealed abstract class JValue { + + def valueType: String + + def getBoolean: Option[Boolean] = None + def getString: Option[String] = None + def getInt: Option[Int] = None + def getLong: Option[Long] = None + def getDouble: Option[Double] = None + def getBigInt: Option[BigInt] = None + def getBigDecimal: Option[BigDecimal] = None + + def asBoolean: Boolean = throw new WrongValueException("boolean", valueType) + def asString: String = throw new WrongValueException("string", valueType) + def asInt: Int = throw new WrongValueException("number", valueType) + def asLong: Long = throw new WrongValueException("number", valueType) + def asDouble: Double = throw new WrongValueException("number", valueType) + def asBigInt: BigInt = throw new WrongValueException("number", valueType) + def asBigDecimal: BigDecimal = throw new WrongValueException("number", valueType) + + def get(i: Int): JValue = JNull + def set(i: Int, v: JValue): Unit = throw new WrongValueException("array", valueType) + + def get(s: String): JValue = JNull + def set(s: String, v: JValue): Unit = throw new WrongValueException("object", valueType) + def remove(s: String): Option[JValue] = None + + final def atomic: Option[JAtom] = + this match { + case v: JAtom => Some(v) + case _ => None + } + + final def isNull: Boolean = + this == JNull + + final def nonNull: Boolean = + this != JNull + + final def render(): String = + CanonicalRenderer.render(this) + + final def render(r: Renderer): String = + r.render(this) + + override def toString: String = + CanonicalRenderer.render(this) +} + +object JValue { + implicit val facade: Facade[JValue] = JawnFacade +} + +sealed abstract class JAtom extends JValue { + def fold[A](f1: String => A, f2: Double => A, f3: Boolean => A, f4: => A): A = + this match { + case JString(s) => f1(s) + case v: JNum => f2(v.asDouble) + case JTrue => f3(true) + case JFalse => f3(false) + case JNull => f4 + } +} + +case object JNull extends JAtom { + final def valueType: String = "null" +} + +sealed abstract class JBool extends JAtom { + final def valueType: String = "boolean" + final override def getBoolean: Option[Boolean] = Some(this == JTrue) + final override def asBoolean: Boolean = this == JTrue +} + +object JBool { + final val True: JBool = JTrue + final val False: JBool = JFalse + + final def apply(b: Boolean): JBool = if (b) JTrue else JFalse +} + +case object JTrue extends JBool +case object JFalse extends JBool + +case class JString(s: String) extends JAtom { + final def valueType: String = "string" + final override def getString: Option[String] = Some(s) + final override def asString: String = s +} + +object JString { + final val empty = JString("") +} + +sealed abstract class JNum extends JAtom { + final def valueType: String = "number" +} + +object JNum { self => + + /** + * Create a JNum from a Long. + * + * This is identical to calling the LongNum(_) constructor. + */ + final def apply(n: Long): JNum = + LongNum(n) + + /** + * Create a JNum from a Double. + * + * This factory constructor performs some error-checking (ensures + * that the given value is a finite Double). If you have already + * done this error-checking, you can use the DoubleNum(_) or + * DeferNum(_) constructors directly. + */ + final def apply(n: Double): JNum = + if (isNaN(n) || isInfinite(n)) throw new InvalidNumException(n.toString) + else DoubleNum(n) + + /** + * Create a JNum from a String. + * + * This factory constructor validates the string (essentially, + * parsing it as a JSON value). If you are already sure this string + * is a valid JSON number, you can use the DeferLong(_) or + * DeferNum(_) constructors directly. + */ + final def apply(s: String): JNum = + JParser.parseUnsafe(s) match { + case jnum: JNum => jnum + case _ => throw new InvalidNumException(s) + } + + final def hybridEq(x: Long, y: Double): Boolean = { + val z = x.toDouble + y == z && z.toLong == x + } + + final val zero: JNum = LongNum(0) + final val one: JNum = LongNum(1) +} + +case class LongNum(n: Long) extends JNum { + + final override def getInt: Option[Int] = Some(n.toInt) + final override def getLong: Option[Long] = Some(n) + final override def getDouble: Option[Double] = Some(n.toDouble) + final override def getBigInt: Option[BigInt] = Some(BigInt(n)) + final override def getBigDecimal: Option[BigDecimal] = Some(BigDecimal(n)) + + final override def asInt: Int = n.toInt + final override def asLong: Long = n + final override def asDouble: Double = n.toDouble + final override def asBigInt: BigInt = BigInt(n) + final override def asBigDecimal: BigDecimal = BigDecimal(n) + + final override def hashCode: Int = n.## + + final override def equals(that: Any): Boolean = + that match { + case LongNum(n2) => n == n2 + case DoubleNum(n2) => JNum.hybridEq(n, n2) + case jn: JNum => jn == this + case _ => false + } +} + +case class DoubleNum(n: Double) extends JNum { + + final override def getInt: Option[Int] = Some(n.toInt) + final override def getLong: Option[Long] = Some(n.toLong) + final override def getDouble: Option[Double] = Some(n) + final override def getBigInt: Option[BigInt] = Some(BigDecimal(n).toBigInt) + final override def getBigDecimal: Option[BigDecimal] = Some(BigDecimal(n)) + + final override def asInt: Int = n.toInt + final override def asLong: Long = n.toLong + final override def asDouble: Double = n + final override def asBigInt: BigInt = BigDecimal(n).toBigInt + final override def asBigDecimal: BigDecimal = BigDecimal(n) + + final override def hashCode: Int = n.## + + final override def equals(that: Any): Boolean = + that match { + case LongNum(n2) => JNum.hybridEq(n2, n) + case DoubleNum(n2) => n == n2 + case jn: JNum => jn == this + case _ => false + } +} + +case class DeferLong(s: String) extends JNum { + + lazy val n: Long = util.parseLongUnsafe(s) + + final override def getInt: Option[Int] = Some(n.toInt) + final override def getLong: Option[Long] = Some(n) + final override def getDouble: Option[Double] = Some(n.toDouble) + final override def getBigInt: Option[BigInt] = Some(BigInt(s)) + final override def getBigDecimal: Option[BigDecimal] = Some(BigDecimal(s)) + + final override def asInt: Int = n.toInt + final override def asLong: Long = n + final override def asDouble: Double = n.toDouble + final override def asBigInt: BigInt = BigInt(s) + final override def asBigDecimal: BigDecimal = BigDecimal(s) + + final override def hashCode: Int = n.## + + final override def equals(that: Any): Boolean = + that match { + case LongNum(n2) => n == n2 + case DoubleNum(n2) => JNum.hybridEq(n, n2) + case jn: DeferLong => n == jn.asLong + case jn: DeferNum => JNum.hybridEq(n, jn.asDouble) + case _ => false + } +} + +case class DeferNum(s: String) extends JNum { + + lazy val n: Double = java.lang.Double.parseDouble(s) + + final override def getInt: Option[Int] = Some(n.toInt) + final override def getLong: Option[Long] = Some(util.parseLongUnsafe(s)) + final override def getDouble: Option[Double] = Some(n) + final override def getBigInt: Option[BigInt] = Some(BigDecimal(s).toBigInt) + final override def getBigDecimal: Option[BigDecimal] = Some(BigDecimal(s)) + + final override def asInt: Int = n.toInt + final override def asLong: Long = util.parseLongUnsafe(s) + final override def asDouble: Double = n + final override def asBigInt: BigInt = BigDecimal(s).toBigInt + final override def asBigDecimal: BigDecimal = BigDecimal(s) + + final override def hashCode: Int = n.## + + final override def equals(that: Any): Boolean = + that match { + case LongNum(n2) => JNum.hybridEq(n2, n) + case DoubleNum(n2) => n == n2 + case jn: DeferLong => JNum.hybridEq(jn.asLong, n) + case jn: DeferNum => n == jn.asDouble + case _ => false + } +} + +case class JArray(vs: Array[JValue]) extends JValue { + final def valueType: String = "array" + + final override def get(i: Int): JValue = + if (0 <= i && i < vs.length) vs(i) else JNull + + final override def set(i: Int, v: JValue): Unit = + vs(i) = v + + final override def hashCode: Int = MurmurHash3.arrayHash(vs) + + final override def equals(that: Any): Boolean = + that match { + case JArray(vs2) => + if (vs.length != vs2.length) return false + var i = 0 + while (i < vs.length) { + if (vs(i) != vs2(i)) return false + i += 1 + } + true + case _ => + false + } +} + +object JArray { self => + final def empty: JArray = + JArray(new Array[JValue](0)) + + final def fromSeq(js: Seq[JValue]): JArray = + JArray(js.toArray) +} + +case class JObject(vs: mutable.Map[String, JValue]) extends JValue { + final def valueType: String = "object" + + final override def get(k: String): JValue = + vs.getOrElse(k, JNull) + + final override def set(k: String, v: JValue): Unit = + vs.put(k, v) + + final override def remove(k: String): Option[JValue] = + vs.remove(k) +} + +object JObject { self => + final def empty: JObject = + JObject(mutable.Map.empty) + + final def fromSeq(js: Seq[(String, JValue)]): JObject = + JObject(mutable.Map(js: _*)) +} diff --git a/scalaplugin/src/test/resource/jawn/ast/src/main/scala/jawn/ast/JawnFacade.scala b/scalaplugin/src/test/resource/jawn/ast/src/main/scala/jawn/ast/JawnFacade.scala new file mode 100644 index 00000000..a2d2d711 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/ast/src/main/scala/jawn/ast/JawnFacade.scala @@ -0,0 +1,51 @@ +package jawn +package ast + +import scala.collection.mutable + +object JawnFacade extends Facade[JValue] { + + final val jnull = JNull + final val jfalse = JFalse + final val jtrue = JTrue + + final def jnum(s: CharSequence, decIndex: Int, expIndex: Int): JValue = + if (decIndex == -1 && expIndex == -1) { + DeferLong(s.toString) + } else { + DeferNum(s.toString) + } + + final def jstring(s: CharSequence): JValue = + JString(s.toString) + + final def singleContext(): FContext[JValue] = + new FContext[JValue] { + var value: JValue = _ + def add(s: CharSequence) { value = JString(s.toString) } + def add(v: JValue) { value = v } + def finish: JValue = value + def isObj: Boolean = false + } + + final def arrayContext(): FContext[JValue] = + new FContext[JValue] { + val vs = mutable.ArrayBuffer.empty[JValue] + def add(s: CharSequence) { vs.append(JString(s.toString)) } + def add(v: JValue) { vs.append(v) } + def finish: JValue = JArray(vs.toArray) + def isObj: Boolean = false + } + + final def objectContext(): FContext[JValue] = + new FContext[JValue] { + var key: String = null + val vs = mutable.Map.empty[String, JValue] + def add(s: CharSequence): Unit = + if (key == null) { key = s.toString } else { vs(key.toString) = JString(s.toString); key = null } + def add(v: JValue): Unit = + { vs(key) = v; key = null } + def finish = JObject(vs) + def isObj = true + } +} diff --git a/scalaplugin/src/test/resource/jawn/ast/src/main/scala/jawn/ast/Renderer.scala b/scalaplugin/src/test/resource/jawn/ast/src/main/scala/jawn/ast/Renderer.scala new file mode 100644 index 00000000..3b2d9103 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/ast/src/main/scala/jawn/ast/Renderer.scala @@ -0,0 +1,101 @@ +package jawn +package ast + +import scala.annotation.switch +import scala.collection.mutable +import scala.util.Sorting + +sealed trait Renderer { + final def render(jv: JValue): String = { + val sb = new StringBuilder + render(sb, 0, jv) + sb.toString + } + + final def render(sb: StringBuilder, depth: Int, jv: JValue): Unit = + jv match { + case JNull => sb.append("null") + case JTrue => sb.append("true") + case JFalse => sb.append("false") + case LongNum(n) => sb.append(n.toString) + case DoubleNum(n) => sb.append(n.toString) + case DeferNum(s) => sb.append(s) + case DeferLong(s) => sb.append(s) + case JString(s) => renderString(sb, s) + case JArray(vs) => renderArray(sb, depth, vs) + case JObject(vs) => renderObject(sb, depth, canonicalizeObject(vs)) + } + + def canonicalizeObject(vs: mutable.Map[String, JValue]): Iterator[(String, JValue)] + + def renderString(sb: StringBuilder, s: String): Unit + + final def renderArray(sb: StringBuilder, depth: Int, vs: Array[JValue]): Unit = { + if (vs.isEmpty) return { sb.append("[]"); () } + sb.append("[") + render(sb, depth + 1, vs(0)) + var i = 1 + while (i < vs.length) { + sb.append(",") + render(sb, depth + 1, vs(i)) + i += 1 + } + sb.append("]") + } + + final def renderObject(sb: StringBuilder, depth: Int, it: Iterator[(String, JValue)]): Unit = { + if (!it.hasNext) return { sb.append("{}"); () } + val (k0, v0) = it.next + sb.append("{") + renderString(sb, k0) + sb.append(":") + render(sb, depth + 1, v0) + while (it.hasNext) { + val (k, v) = it.next + sb.append(",") + renderString(sb, k) + sb.append(":") + render(sb, depth + 1, v) + } + sb.append("}") + } + + final def escape(sb: StringBuilder, s: String, unicode: Boolean): Unit = { + sb.append('"') + var i = 0 + val len = s.length + while (i < len) { + (s.charAt(i): @switch) match { + case '"' => sb.append("\\\"") + case '\\' => sb.append("\\\\") + case '\b' => sb.append("\\b") + case '\f' => sb.append("\\f") + case '\n' => sb.append("\\n") + case '\r' => sb.append("\\r") + case '\t' => sb.append("\\t") + case c => + if (c < ' ' || (c > '~' && unicode)) sb.append("\\u%04x" format c.toInt) + else sb.append(c) + } + i += 1 + } + sb.append('"') + } +} + +object CanonicalRenderer extends Renderer { + def canonicalizeObject(vs: mutable.Map[String, JValue]): Iterator[(String, JValue)] = { + val keys = vs.keys.toArray + Sorting.quickSort(keys) + keys.iterator.map(k => (k, vs(k))) + } + def renderString(sb: StringBuilder, s: String): Unit = + escape(sb, s, true) +} + +object FastRenderer extends Renderer { + def canonicalizeObject(vs: mutable.Map[String, JValue]): Iterator[(String, JValue)] = + vs.iterator + def renderString(sb: StringBuilder, s: String): Unit = + escape(sb, s, false) +} diff --git a/scalaplugin/src/test/resource/jawn/ast/src/test/scala/jawn/ArbitraryUtil.scala b/scalaplugin/src/test/resource/jawn/ast/src/test/scala/jawn/ArbitraryUtil.scala new file mode 100644 index 00000000..6fdb8fbe --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/ast/src/test/scala/jawn/ArbitraryUtil.scala @@ -0,0 +1,49 @@ +package jawn +package ast + +import org.scalacheck._ +import Gen._ +import Arbitrary.arbitrary + +object ArbitraryUtil { + + // JSON doesn't allow NaN, PositiveInfinity, or NegativeInfinity + def isFinite(n: Double): Boolean = + !java.lang.Double.isNaN(n) && !java.lang.Double.isInfinite(n) + + val jnull = Gen.const(JNull) + val jboolean = Gen.oneOf(JTrue :: JFalse :: Nil) + val jlong = arbitrary[Long].map(LongNum(_)) + val jdouble = arbitrary[Double].filter(isFinite).map(DoubleNum(_)) + val jstring = arbitrary[String].map(JString(_)) + + // Totally unscientific atom frequencies. + val jatom: Gen[JAtom] = + Gen.frequency( + (1, jnull), + (8, jboolean), + (8, jlong), + (8, jdouble), + (16, jstring)) + + // Use lvl to limit the depth of our jvalues. + // Otherwise we will end up with SOE real fast. + + val MaxLevel: Int = 3 + + def jarray(lvl: Int): Gen[JArray] = + Gen.containerOf[Array, JValue](jvalue(lvl + 1)).map(JArray(_)) + + def jitem(lvl: Int): Gen[(String, JValue)] = + for { s <- arbitrary[String]; j <- jvalue(lvl) } yield (s, j) + + def jobject(lvl: Int): Gen[JObject] = + Gen.containerOf[Vector, (String, JValue)](jitem(lvl + 1)).map(JObject.fromSeq) + + def jvalue(lvl: Int = 0): Gen[JValue] = + if (lvl >= MaxLevel) jatom + else Gen.frequency((16, jatom), (1, jarray(lvl)), (2, jobject(lvl))) + + implicit lazy val arbitraryJValue: Arbitrary[JValue] = + Arbitrary(jvalue()) +} diff --git a/scalaplugin/src/test/resource/jawn/ast/src/test/scala/jawn/AstTest.scala b/scalaplugin/src/test/resource/jawn/ast/src/test/scala/jawn/AstTest.scala new file mode 100644 index 00000000..3ec7373e --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/ast/src/test/scala/jawn/AstTest.scala @@ -0,0 +1,79 @@ +package jawn +package ast + +import org.scalatest._ +import org.scalatest.prop._ + +import scala.collection.mutable +import scala.util.{Try, Success} + +import ArbitraryUtil._ + +class AstTest extends PropSpec with Matchers with PropertyChecks { + + property("calling .get never crashes") { + forAll { (v: JValue, s: String, i: Int) => + Try(v.get(i).get(s)).isSuccess shouldBe true + Try(v.get(s).get(i)).isSuccess shouldBe true + Try(v.get(i).get(i)).isSuccess shouldBe true + Try(v.get(s).get(s)).isSuccess shouldBe true + } + } + + property(".getX and .asX agree") { + forAll { (v: JValue) => + v.getBoolean shouldBe Try(v.asBoolean).toOption + v.getString shouldBe Try(v.asString).toOption + v.getInt shouldBe Try(v.asInt).toOption + v.getLong shouldBe Try(v.asLong).toOption + v.getDouble shouldBe Try(v.asDouble).toOption + v.getBigInt shouldBe Try(v.asBigInt).toOption + v.getBigDecimal shouldBe Try(v.asBigDecimal).toOption + } + } + + property(".getBoolean") { + forAll((b: Boolean) => JBool(b).getBoolean shouldBe Some(b)) + } + + property(".getString") { + forAll((s: String) => JString(s).getString shouldBe Some(s)) + } + + property(".getInt") { + forAll { (n: Int) => + JNum(n).getInt shouldBe Some(n) + JParser.parseUnsafe(n.toString).getInt shouldBe Some(n) + } + } + + property(".getLong") { + forAll { (n: Long) => + JNum(n).getLong shouldBe Some(n) + JParser.parseUnsafe(n.toString).getLong shouldBe Some(n) + } + } + + property(".getDouble") { + forAll { (n: Double) => + JNum(n).getDouble shouldBe Some(n) + JParser.parseUnsafe(n.toString).getDouble shouldBe Some(n) + } + } + + property(".getBigInt") { + forAll { (n: BigInt) => + JNum(n.toString).getBigInt shouldBe Some(n) + JParser.parseUnsafe(n.toString).getBigInt shouldBe Some(n) + } + } + + property(".getBigDecimal") { + forAll { (n: BigDecimal) => + if (Try(BigDecimal(n.toString)) == Success(n)) { + JNum(n.toString).getBigDecimal shouldBe Some(n) + JParser.parseUnsafe(n.toString).getBigDecimal shouldBe Some(n) + } + } + } +} diff --git a/scalaplugin/src/test/resource/jawn/ast/src/test/scala/jawn/ParseCheck.scala b/scalaplugin/src/test/resource/jawn/ast/src/test/scala/jawn/ParseCheck.scala new file mode 100644 index 00000000..a5a5aa18 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/ast/src/test/scala/jawn/ParseCheck.scala @@ -0,0 +1,169 @@ +package jawn +package ast + +import org.scalatest._ +import org.scalatest.prop._ +import org.scalacheck.Arbitrary._ +import org.scalacheck._ +import Gen._ +import Arbitrary.arbitrary + +import scala.collection.mutable +import scala.util.{Try, Success} + +import jawn.parser.TestUtil + +import ArbitraryUtil._ + +class AstCheck extends PropSpec with Matchers with PropertyChecks { + + // so it's only one property, but it exercises: + // + // * parsing from strings + // * rendering jvalues to string + // * jvalue equality + // + // not bad. + property("idempotent parsing/rendering") { + forAll { value1: JValue => + val json1 = CanonicalRenderer.render(value1) + val value2 = JParser.parseFromString(json1).get + val json2 = CanonicalRenderer.render(value2) + json2 shouldBe json1 + json2.## shouldBe json1.## + + value1 shouldBe value2 + value1.## shouldBe value2.## + + TestUtil.withTemp(json1) { t => + JParser.parseFromFile(t).get shouldBe value2 + } + } + } + + property("string encoding/decoding") { + forAll { s: String => + val jstr1 = JString(s) + val json1 = CanonicalRenderer.render(jstr1) + val jstr2 = JParser.parseFromString(json1).get + val json2 = CanonicalRenderer.render(jstr2) + jstr2 shouldBe jstr1 + json2 shouldBe json1 + json2.## shouldBe json1.## + } + } + + property("string/charSequence parsing") { + forAll { value: JValue => + val s = CanonicalRenderer.render(value) + val j1 = JParser.parseFromString(s) + val cs = java.nio.CharBuffer.wrap(s.toCharArray) + val j2 = JParser.parseFromCharSequence(cs) + j1 shouldBe j2 + j1.## shouldBe j2.## + } + } + + implicit val facade = JawnFacade + + val percs = List(0.0, 0.2, 0.4, 0.8, 1.0) + + def checkRight(r: Either[ParseException, Seq[JValue]]): Seq[JValue] = { + r.isRight shouldBe true + val Right(vs) = r + vs + } + + def splitIntoSegments(json: String): List[String] = + if (json.length >= 8) { + val offsets = percs.map(n => (json.length * n).toInt) + val pairs = offsets zip offsets.drop(1) + pairs.map { case (i, j) => json.substring(i, j) } + } else { + json :: Nil + } + + def parseSegments(p: AsyncParser[JValue], segments: List[String]): Seq[JValue] = + segments.foldLeft(List.empty[JValue]) { (rs, s) => + rs ++ checkRight(p.absorb(s)) + } ++ checkRight(p.finish()) + + import AsyncParser.{UnwrapArray, ValueStream, SingleValue} + + property("async multi") { + val data = "[1,2,3][4,5,6]" + val p = AsyncParser[JValue](ValueStream) + val res0 = p.absorb(data) + val res1 = p.finish + //println((res0, res1)) + true + } + + property("async parsing") { + forAll { (v: JValue) => + val json = CanonicalRenderer.render(v) + val segments = splitIntoSegments(json) + val parsed = parseSegments(AsyncParser[JValue](SingleValue), segments) + parsed shouldBe List(v) + } + } + + property("async unwrapping") { + forAll { (vs0: List[Int]) => + val vs = vs0.map(LongNum(_)) + val arr = JArray(vs.toArray) + val json = CanonicalRenderer.render(arr) + val segments = splitIntoSegments(json) + parseSegments(AsyncParser[JValue](UnwrapArray), segments) shouldBe vs + } + } + + property("unicode string round-trip") { + forAll { (s: String) => + JParser.parseFromString(JString(s).render(FastRenderer)) shouldBe Success(JString(s)) + } + } + + property("if x == y, then x.## == y.##") { + forAll { (x: JValue, y: JValue) => + if (x == y) x.## shouldBe y.## + } + } + + property("ignore trailing zeros") { + forAll { (n: Int) => + val s = n.toString + val n1 = LongNum(n) + val n2 = DoubleNum(n) + + def check(j: JValue) { + j shouldBe n1; n1 shouldBe j + j shouldBe n2; n2 shouldBe j + } + + check(DeferNum(s)) + check(DeferNum(s + ".0")) + check(DeferNum(s + ".00")) + check(DeferNum(s + ".000")) + check(DeferNum(s + "e0")) + check(DeferNum(s + ".0e0")) + } + } + + property("large strings") { + val M = 1000000 + val q = "\"" + + val s0 = ("x" * (40 * M)) + val e0 = q + s0 + q + TestUtil.withTemp(e0) { t => + JParser.parseFromFile(t).filter(_ == JString(s0)).isSuccess shouldBe true + } + + val s1 = "\\" * (20 * M) + val e1 = q + s1 + s1 + q + TestUtil.withTemp(e1) { t => + JParser.parseFromFile(t).filter(_ == JString(s1)).isSuccess shouldBe true + } + } +} diff --git a/scalaplugin/src/test/resource/jawn/benchmark/build.sbt b/scalaplugin/src/test/resource/jawn/benchmark/build.sbt new file mode 100644 index 00000000..7cb15b12 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/benchmark/build.sbt @@ -0,0 +1,21 @@ +name := "jawn-benchmarks" + +javaOptions in run += "-Xmx6G" + +libraryDependencies ++= Seq( + "io.argonaut" %% "argonaut" % "6.2", + "org.json4s" %% "json4s-native" % "3.5.2", + "org.json4s" %% "json4s-jackson" % "3.5.2", + "com.typesafe.play" %% "play-json" % "2.5.15", + "com.rojoma" %% "rojoma-json" % "2.4.3", + "com.rojoma" %% "rojoma-json-v3" % "3.7.2", + "io.spray" %% "spray-json" % "1.3.3", + "org.parboiled" %% "parboiled" % "2.1.4", + "com.fasterxml.jackson.core" % "jackson-annotations" % "2.8.4", + "com.fasterxml.jackson.core" % "jackson-core" % "2.8.4", + "com.fasterxml.jackson.core" % "jackson-databind" % "2.8.4", + "com.google.code.gson" % "gson" % "2.8.1" +) + +// enable forking in run +fork in run := true diff --git a/scalaplugin/src/test/resource/jawn/benchmark/src/main/scala/jawn/JmhBenchmarks.scala b/scalaplugin/src/test/resource/jawn/benchmark/src/main/scala/jawn/JmhBenchmarks.scala new file mode 100644 index 00000000..bc56f9f6 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/benchmark/src/main/scala/jawn/JmhBenchmarks.scala @@ -0,0 +1,120 @@ +package jawn +package benchmark + +import java.io.{BufferedReader, File, FileInputStream, FileReader} +import java.util.concurrent.TimeUnit +import org.openjdk.jmh.annotations._ +import scala.collection.mutable + +@State(Scope.Benchmark) +@BenchmarkMode(Array(Mode.AverageTime)) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +abstract class JmhBenchmarks(name: String) { + val path: String = s"src/main/resources/$name" + + def load(path: String): String = { + val file = new File(path) + val bytes = new Array[Byte](file.length.toInt) + val fis = new FileInputStream(file) + fis.read(bytes) + new String(bytes, "UTF-8") + } + + def reader(path: String): FileReader = + new FileReader(new File(path)) + + def buffered(path: String): BufferedReader = + new BufferedReader(new FileReader(new File(path))) + + @Benchmark + def jawnCheckSyntax() = + jawn.Syntax.checkString(load(path)) + + @Benchmark + def jawnParse() = + jawn.ast.JParser.parseFromFile(new File(path)).get + + @Benchmark + def jawnStringParse() = + jawn.ast.JParser.parseFromString(load(path)).get +} + +trait OtherBenchmarks { self: JmhBenchmarks => + @Benchmark + def json4sJacksonParse() = { + import org.json4s._ + import org.json4s.jackson.JsonMethods._ + parse(load(path)) + } + + @Benchmark + def playParse() = + play.api.libs.json.Json.parse(load(path)) + + @Benchmark + def rojomaV3Parse() = + com.rojoma.json.v3.io.JsonReader.fromReader(reader(path), blockSize = 100000) + + @Benchmark + def argonautParse() = + argonaut.Parse.parse(load(path)) + + @Benchmark + def sprayParse() = + spray.json.JsonParser(load(path)) + + @Benchmark + def parboiledJsonParse() = + new ParboiledParser(load(path)).Json.run().get + + @Benchmark + def jacksonParse() = { + import com.fasterxml.jackson.databind.ObjectMapper + import com.fasterxml.jackson.databind.JsonNode + new ObjectMapper().readValue(new File(path), classOf[JsonNode]) + } + + @Benchmark + def gsonParse() = + new com.google.gson.JsonParser().parse(buffered(path)) + + // don't bother benchmarking jawn + external asts by default + + // @Benchmark + // def json4sJawnParse() = + // jawn.support.json4s.Parser.parseFromFile(new File(path)).get + // + // @Benchmark + // def rojomaV3JawnParse() = + // jawn.support.rojoma.v3.Parser.parseFromFile(new File(path)).get + // + // @Benchmark + // def argonautJawnParse() = + // jawn.support.argonaut.Parser.parseFromFile(new File(path)).get + // + // @Benchmark + // def sprayJawnParse() = + // jawn.support.spray.Parser.parseFromFile(new File(path)).get + + // native json4s parser is really, really slow, so it's disabled by default. + + // @Benchmark + // def json4sNativeParse() = { + // import org.json4s._ + // import org.json4s.native.JsonMethods._ + // parse(load(path)) + // } +} + +class Qux2Bench extends JmhBenchmarks("qux2.json") with OtherBenchmarks +class Bla25Bench extends JmhBenchmarks("bla25.json") with OtherBenchmarks +class CountriesBench extends JmhBenchmarks("countries.geo.json") with OtherBenchmarks +class Ugh10kBench extends JmhBenchmarks("ugh10k.json") with OtherBenchmarks + +class JawnOnlyQux2Bench extends JmhBenchmarks("qux2.json") +class JawnOnlyBla25Bench extends JmhBenchmarks("bla25.json") +class JawnOnlyCountriesBench extends JmhBenchmarks("countries.geo.json") +class JawnOnlyUgh10kBench extends JmhBenchmarks("ugh10k.json") + +// // from https://github.com/zemirco/sf-city-lots-json +// class CityLotsBench extends JmhBenchmarks("citylots.json") diff --git a/scalaplugin/src/test/resource/jawn/benchmark/src/main/scala/jawn/Parboiled.scala b/scalaplugin/src/test/resource/jawn/benchmark/src/main/scala/jawn/Parboiled.scala new file mode 100644 index 00000000..bd5fed18 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/benchmark/src/main/scala/jawn/Parboiled.scala @@ -0,0 +1,105 @@ +package jawn.benchmark + +/* + * Copyright (C) 2009-2013 Mathias Doenitz, Alexander Myltsev + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import scala.annotation.switch +import org.parboiled2._ +import spray.json.{ParserInput => _, _} + +/** + * This is a feature-complete JSON parser implementation that almost directly + * models the JSON grammar presented at http://www.json.org as a parboiled2 PEG parser. + */ +class ParboiledParser(val input: ParserInput) extends Parser with StringBuilding { + import CharPredicate.{Digit, Digit19, HexDigit} + import ParboiledParser._ + + // the root rule + def Json = rule { WhiteSpace ~ Value ~ EOI } + + def JsonObject: Rule1[JsObject] = rule { + ws('{') ~ zeroOrMore(Pair).separatedBy(ws(',')) ~ ws('}') ~> ((fields: Seq[JsField]) => JsObject(fields :_*)) + } + + def Pair = rule { JsonStringUnwrapped ~ ws(':') ~ Value ~> ((_, _)) } + + def Value: Rule1[JsValue] = rule { + // as an optimization of the equivalent rule: + // JsonString | JsonNumber | JsonObject | JsonArray | JsonTrue | JsonFalse | JsonNull + // we make use of the fact that one-char lookahead is enough to discriminate the cases + run { + (cursorChar: @switch) match { + case '"' => JsonString + case '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | '-' => JsonNumber + case '{' => JsonObject + case '[' => JsonArray + case 't' => JsonTrue + case 'f' => JsonFalse + case 'n' => JsonNull + case _ => MISMATCH + } + } + } + + def JsonString = rule { JsonStringUnwrapped ~> (JsString(_)) } + + def JsonStringUnwrapped = rule { '"' ~ clearSB() ~ Characters ~ ws('"') ~ push(sb.toString) } + + def JsonNumber = rule { capture(Integer ~ optional(Frac) ~ optional(Exp)) ~> (JsNumber(_)) ~ WhiteSpace } + + def JsonArray = rule { ws('[') ~ zeroOrMore(Value).separatedBy(ws(',')) ~ ws(']') ~> (JsArray(_ :_*)) } + + def Characters = rule { zeroOrMore(NormalChar | '\\' ~ EscapedChar) } + + def NormalChar = rule { !QuoteBackslash ~ ANY ~ appendSB() } + + def EscapedChar = rule ( + QuoteSlashBackSlash ~ appendSB() + | 'b' ~ appendSB('\b') + | 'f' ~ appendSB('\f') + | 'n' ~ appendSB('\n') + | 'r' ~ appendSB('\r') + | 't' ~ appendSB('\t') + | Unicode ~> { code => sb.append(code.asInstanceOf[Char]); () } + ) + + def Unicode = rule { 'u' ~ capture(HexDigit ~ HexDigit ~ HexDigit ~ HexDigit) ~> (java.lang.Integer.parseInt(_, 16)) } + + def Integer = rule { optional('-') ~ (Digit19 ~ Digits | Digit) } + + def Digits = rule { oneOrMore(Digit) } + + def Frac = rule { "." ~ Digits } + + def Exp = rule { ignoreCase('e') ~ optional(anyOf("+-")) ~ Digits } + + def JsonTrue = rule { "true" ~ WhiteSpace ~ push(JsTrue) } + + def JsonFalse = rule { "false" ~ WhiteSpace ~ push(JsFalse) } + + def JsonNull = rule { "null" ~ WhiteSpace ~ push(JsNull) } + + def WhiteSpace = rule { zeroOrMore(WhiteSpaceChar) } + + def ws(c: Char) = rule { c ~ WhiteSpace } +} + +object ParboiledParser { + val WhiteSpaceChar = CharPredicate(" \n\r\t\f") + val QuoteBackslash = CharPredicate("\"\\") + val QuoteSlashBackSlash = QuoteBackslash ++ "/" +} diff --git a/scalaplugin/src/test/resource/jawn/benchmark/src/main/scala/jawn/ParseLongBench.scala b/scalaplugin/src/test/resource/jawn/benchmark/src/main/scala/jawn/ParseLongBench.scala new file mode 100644 index 00000000..97e8e6a8 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/benchmark/src/main/scala/jawn/ParseLongBench.scala @@ -0,0 +1,133 @@ +package jawn +package benchmark + +import java.io.{BufferedReader, File, FileInputStream, FileReader} +import java.util.concurrent.TimeUnit +import org.openjdk.jmh.annotations._ +import scala.collection.mutable + +case class Slice(s: String, begin: Int, limit: Int) extends CharSequence { + val length: Int = limit - begin + def charAt(i: Int): Char = s.charAt(begin + i) + def subSequence(start: Int, end: Int): Slice = + Slice(s, begin + start, Math.min(end + begin, limit)) + override def toString: String = + s.substring(begin, limit) +} + +@State(Scope.Benchmark) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +class ParseLongBench { + + val longs: Array[Long] = Array( + -1346837161442476189L, -4666345991836441070L, 4868830844043235709L, + 2992690405064579158L, -2017521011608078634L, -3039682866169364757L, + 8997687047891586260L, 5932727796276454607L, 4062739618560250554L, + 8668950167358198490L, -8565613821858118870L, 8049785848575684314L, + -580831266940599830L, -3593199367295538945L, 8374322595267797482L, + 3088261552516619129L, -6879203747403593851L, -1842900848925949857L, + 4484592876047641351L, 5182973278356955602L, -6840392853855436945L, + -4176340556015032222L, -536379174926548619L, 6343722878919863216L, + 1557757008211571405L, -334093799456298669L, 619602023052756397L, + 6904874397154297343L, -4332034907782234995L, -8767842695446545180L, + -6127250063205613011L, 6902212562850963795L, 4778607575334665692L, + 7674074815344809639L, -3834944692798167050L, 7406081418831471202L, + -9126886315356724563L, 8093878176633322645L, 2471547025788214028L, + -5018828829942988155L, -6676531171364391367L, 8189793226936659851L, + 7150026713387306746L, -6065566098373722052L, 3281133763697608570L, + 957103694526079944L, -3009447279791131829L, -1995600795755716697L, + 2361055030313262510L, -4312828282749171343L, 8836216125516165138L, + 5548785979447786253L, 8567551485822958810L, 5931896003625723150L, + 3472058092439106147L, 4363240277904515929L, -2999484068697753019L, + -8285358702782547958L, -2407429647076308777L, 4411565001760018584L, + 792384115860070648L, 3328145302561962294L, -2377559446421434356L, + -7837698939558960516L, -565806101451282875L, -4792610084643070650L, + 2713520205731589923L, -6521104721472605988L, 5037187811345411645L, + 3866939564433764178L, -3851229228204678079L, -8171137274242372558L, + -14713951794749384L, 2061783257002637655L, -7375571393873059570L, + 7402007407273053723L, -5104318069025846447L, -8956415532448219980L, + 4904595193891993401L, 5396360181536889307L, -8043917553767343384L, + -3666269817017255250L, -6535587792359353103L, -4553034734642385706L, + -7544140164897268962L, 2468330113904053484L, 5790319365381968237L, + -2734383156062609640L, -4831208471935595172L, 4502079643250626043L, + 4778622151522470246L, 7233054223498326990L, 5833883346008509644L, + -8013495378054295093L, 2944606201054530456L, -8608231828651976245L, + -6957117814546267426L, -4744827311133020624L, 2640030216500286789L, + 8343959867315747844L) + + val strs: Array[CharSequence] = + longs.map(_.toString) + + val seqs: Array[CharSequence] = + longs.map { n => + val prefix = "x" * (n & 63).toInt + val suffix = "y" * ((n * 7) & 63).toInt + val i = prefix.length + val s = n.toString + Slice(prefix + s + suffix, i, s.length + i) + } + + val str: CharSequence = "23948271429443" + + val seq: CharSequence = Slice("weigjewigjwi23948271429443jgewigjweiwjegiwgjiewjgeiwjg", 12, 26) + + def sumJava(css: Array[CharSequence]): Long = { + var sum: Long = 0 + var i = 0 + while (i < css.length) { + sum += java.lang.Long.parseLong(css(i).toString) + i += 1 + } + sum + } + + def sumStd(css: Array[CharSequence]): Long = { + var sum: Long = 0 + var i = 0 + while (i < css.length) { + sum += css(i).toString.toLong + i += 1 + } + sum + } + + def sumSafe(css: Array[CharSequence]): Long = { + var sum: Long = 0 + var i = 0 + while (i < css.length) { + sum += Util.parseLong(css(i)) + i += 1 + } + sum + } + + def sumUnsafe(css: Array[CharSequence]): Long = { + var sum: Long = 0 + var i = 0 + while (i < css.length) { + sum += Util.parseLongUnsafe(css(i)) + i += 1 + } + sum + } + + @Benchmark def stringArrayJava(): Long = sumJava(strs) + @Benchmark def seqArrayJava(): Long = sumJava(seqs) + @Benchmark def stringValueJava(): Long = java.lang.Long.parseLong(str.toString) + @Benchmark def seqValueJava(): Long = java.lang.Long.parseLong(seq.toString) + + @Benchmark def stringArrayStd(): Long = sumStd(strs) + @Benchmark def seqArrayStd(): Long = sumStd(seqs) + @Benchmark def stringValueStd(): Long = str.toString.toLong + @Benchmark def seqValueStd(): Long = seq.toString.toLong + + @Benchmark def stringArraySafe(): Long = sumSafe(strs) + @Benchmark def seqArraySafe(): Long = sumSafe(seqs) + @Benchmark def stringValueSafe(): Long = Util.parseLong(str) + @Benchmark def seqValueSafe(): Long = Util.parseLong(seq) + + @Benchmark def stringArrayUnsafe(): Long = sumUnsafe(strs) + @Benchmark def seqArrayUnsafe(): Long = sumUnsafe(seqs) + @Benchmark def stringValueUnsafe(): Long = Util.parseLongUnsafe(str) + @Benchmark def seqValueUnsafe(): Long = Util.parseLongUnsafe(seq) +} diff --git a/scalaplugin/src/test/resource/jawn/build.sbt b/scalaplugin/src/test/resource/jawn/build.sbt new file mode 100644 index 00000000..c32403ed --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/build.sbt @@ -0,0 +1,162 @@ +import ReleaseTransformations._ + +lazy val previousJawnVersion = "0.10.4" + +lazy val stableCrossVersions = + Seq("2.10.6", "2.11.11", "2.12.2") + +// we'll support 2.13.0-M1 soon but not yet +lazy val allCrossVersions = + stableCrossVersions + +lazy val benchmarkVersion = + "2.12.2" + +lazy val jawnSettings = Seq( + organization := "org.spire-math", + scalaVersion := "2.12.2", + crossScalaVersions := allCrossVersions, + + mimaPreviousArtifacts := Set(organization.value %% moduleName.value % previousJawnVersion), + + resolvers += Resolver.sonatypeRepo("releases"), + + libraryDependencies ++= + "org.scalatest" %% "scalatest" % "3.0.3" % Test :: + "org.scalacheck" %% "scalacheck" % "1.13.5" % Test :: + Nil, + + scalacOptions ++= + "-deprecation" :: + "-optimize" :: + "-unchecked" :: + Nil, + + licenses += ("MIT", url("http://opensource.org/licenses/MIT")), + homepage := Some(url("http://github.com/non/jawn")), + + // release stuff + releaseCrossBuild := true, + publishMavenStyle := true, + publishArtifact in Test := false, + pomIncludeRepository := Function.const(false), + + publishTo := { + val nexus = "https://oss.sonatype.org/" + if (isSnapshot.value) { + Some("Snapshots" at nexus + "content/repositories/snapshots") + } else { + Some("Releases" at nexus + "service/local/staging/deploy/maven2") + } + }, + + scmInfo := Some(ScmInfo( + browseUrl = url("https://github.com/non/jawn"), + connection = "scm:git:git@github.com:non/jawn.git" + )), + + developers += Developer( + name = "Erik Osheim", + email = "erik@plastic-idolatry.com", + id = "d_m", + url = url("http://github.com/non/") + ), + + releaseProcess := Seq[ReleaseStep]( + checkSnapshotDependencies, + inquireVersions, + runClean, + ReleaseHelper.runCommandAndRemaining("+test"), // formerly runTest + setReleaseVersion, + commitReleaseVersion, + tagRelease, + ReleaseHelper.runCommandAndRemaining("+publishSigned"), + setNextVersion, + commitNextVersion, + ReleaseStep(action = Command.process("sonatypeReleaseAll", _)), + pushChanges)) + +lazy val noPublish = Seq( + publish := {}, + publishLocal := {}, + publishArtifact := false, + mimaPreviousArtifacts := Set()) + +lazy val root = project.in(file(".")) + .aggregate(all.map(Project.projectToRef): _*) + .enablePlugins(CrossPerProjectPlugin) + .disablePlugins(JmhPlugin) + .settings(name := "jawn") + .settings(jawnSettings: _*) + .settings(noPublish: _*) + +lazy val parser = project.in(file("parser")) + .settings(name := "parser") + .settings(moduleName := "jawn-parser") + .settings(jawnSettings: _*) + .disablePlugins(JmhPlugin) + +lazy val util = project.in(file("util")) + .dependsOn(parser % "compile->compile;test->test") + .settings(name := "util") + .settings(moduleName := "jawn-util") + .settings(jawnSettings: _*) + .disablePlugins(JmhPlugin) + +lazy val ast = project.in(file("ast")) + .dependsOn(parser % "compile->compile;test->test") + .dependsOn(util % "compile->compile;test->test") + .settings(name := "ast") + .settings(moduleName := "jawn-ast") + .settings(jawnSettings: _*) + .disablePlugins(JmhPlugin) + +def support(s: String) = + Project(id = s, base = file(s"support/$s")) + .settings(name := (s + "-support")) + .settings(moduleName := "jawn-" + s) + .dependsOn(parser) + .settings(jawnSettings: _*) + .disablePlugins(JmhPlugin) + +lazy val supportArgonaut = support("argonaut") + .settings(crossScalaVersions := stableCrossVersions) + .settings(libraryDependencies += "io.argonaut" %% "argonaut" % "6.2") + +lazy val supportJson4s = support("json4s") + .dependsOn(util) + .settings(crossScalaVersions := stableCrossVersions) + .settings(libraryDependencies += "org.json4s" %% "json4s-ast" % "3.5.2") + +lazy val supportPlay = support("play") + .settings(crossScalaVersions := stableCrossVersions) + .settings(libraryDependencies += (scalaBinaryVersion.value match { + case "2.10" => "com.typesafe.play" %% "play-json" % "2.4.11" + case "2.11" => "com.typesafe.play" %% "play-json" % "2.5.15" + case _ => "com.typesafe.play" %% "play-json" % "2.6.0" + })) + +lazy val supportRojoma = support("rojoma") + .settings(crossScalaVersions := stableCrossVersions) + .settings(libraryDependencies += "com.rojoma" %% "rojoma-json" % "2.4.3") + +lazy val supportRojomaV3 = support("rojoma-v3") + .settings(crossScalaVersions := stableCrossVersions) + .settings(libraryDependencies += "com.rojoma" %% "rojoma-json-v3" % "3.7.2") + +lazy val supportSpray = support("spray") + .settings(crossScalaVersions := stableCrossVersions) + .settings(resolvers += "spray" at "http://repo.spray.io/") + .settings(libraryDependencies += "io.spray" %% "spray-json" % "1.3.3") + +lazy val benchmark = project.in(file("benchmark")) + .dependsOn(all.map(Project.classpathDependency[Project]): _*) + .settings(name := "jawn-benchmark") + .settings(jawnSettings: _*) + .settings(scalaVersion := benchmarkVersion) + .settings(crossScalaVersions := Seq(benchmarkVersion)) + .settings(noPublish: _*) + .enablePlugins(JmhPlugin) + +lazy val all = + Seq(parser, util, ast, supportArgonaut, supportJson4s, supportPlay, supportRojoma, supportRojomaV3, supportSpray) diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/resources/utf8.json b/scalaplugin/src/test/resource/jawn/parser/src/main/resources/utf8.json new file mode 100644 index 00000000..6549eaa0 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/resources/utf8.json @@ -0,0 +1,7 @@ +{ + "copyright": "©", + "accent-e": "é", + "combined-e": "é", + "devenagari": "क्तु", + "math": "𝔊" +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/AsyncParser.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/AsyncParser.scala new file mode 100644 index 00000000..acf770d7 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/AsyncParser.scala @@ -0,0 +1,319 @@ +package jawn + +import scala.annotation.{switch, tailrec} +import scala.math.max +import scala.collection.mutable +import scala.util.control +import java.nio.ByteBuffer + +object AsyncParser { + + sealed abstract class Mode(val start: Int, val value: Int) + case object UnwrapArray extends Mode(-5, 1) + case object ValueStream extends Mode(-1, 0) + case object SingleValue extends Mode(-1, -1) + + def apply[J](mode: Mode = SingleValue): AsyncParser[J] = + new AsyncParser(state = mode.start, curr = 0, stack = Nil, + data = new Array[Byte](131072), len = 0, allocated = 131072, + offset = 0, done = false, streamMode = mode.value) +} + +/** + * AsyncParser is able to parse chunks of data (encoded as + * Option[ByteBuffer] instances) and parse asynchronously. You can + * use the factory methods in the companion object to instantiate an + * async parser. + * + * The async parser's fields are described below: + * + * The (state, curr, stack) triple is used to save and restore parser + * state between async calls. State also helps encode extra + * information when streaming or unwrapping an array. + * + * The (data, len, allocated) triple is used to manage the underlying + * data the parser is keeping track of. As new data comes in, data may + * be expanded if not enough space is available. + * + * The offset parameter is used to drive the outer async parsing. It + * stores similar information to curr but is kept separate to avoid + * "corrupting" our snapshot. + * + * The done parameter is used internally to help figure out when the + * atEof() parser method should return true. This will be set when + * apply(None) is called. + * + * The streamMode parameter controls how the asynchronous parser will + * be handling multiple values. There are three states: + * + * 1: An array is being unwrapped. Normal JSON array rules apply + * (Note that if the outer value observed is not an array, this + * mode will toggle to the -1 mode). + * + * 0: A stream of individual JSON elements separated by whitespace + * are being parsed. We can return each complete element as we + * parse it. + * + * -1: No streaming is occuring. Only a single JSON value is + * allowed. + */ +final class AsyncParser[J] protected[jawn] ( + protected[jawn] var state: Int, + protected[jawn] var curr: Int, + protected[jawn] var stack: List[FContext[J]], + protected[jawn] var data: Array[Byte], + protected[jawn] var len: Int, + protected[jawn] var allocated: Int, + protected[jawn] var offset: Int, + protected[jawn] var done: Boolean, + protected[jawn] var streamMode: Int +) extends ByteBasedParser[J] { + + protected[this] var line = 0 + protected[this] var pos = 0 + protected[this] final def newline(i: Int) { line += 1; pos = i + 1 } + protected[this] final def column(i: Int) = i - pos + + final def copy() = + new AsyncParser(state, curr, stack, data.clone, len, allocated, offset, done, streamMode) + + final def absorb(buf: ByteBuffer)(implicit facade: Facade[J]): Either[ParseException, Seq[J]] = { + done = false + val buflen = buf.limit - buf.position + val need = len + buflen + resizeIfNecessary(need) + buf.get(data, len, buflen) + len = need + churn() + } + + final def absorb(bytes: Array[Byte])(implicit facade: Facade[J]): Either[ParseException, Seq[J]] = + absorb(ByteBuffer.wrap(bytes)) + + final def absorb(s: String)(implicit facade: Facade[J]): Either[ParseException, Seq[J]] = + absorb(ByteBuffer.wrap(s.getBytes(utf8))) + + final def finish()(implicit facade: Facade[J]): Either[ParseException, Seq[J]] = { + done = true + churn() + } + + protected[this] final def resizeIfNecessary(need: Int): Unit = { + // if we don't have enough free space available we'll need to grow our + // data array. we never shrink the data array, assuming users will call + // feed with similarly-sized buffers. + if (need > allocated) { + val doubled = if (allocated < 0x40000000) allocated * 2 else Int.MaxValue + val newsize = max(need, doubled) + val newdata = new Array[Byte](newsize) + System.arraycopy(data, 0, newdata, 0, len) + data = newdata + allocated = newsize + } + } + + /** + * Explanation of the new synthetic states. The parser machinery + * uses positive integers for states while parsing json values. We + * use these negative states to keep track of the async parser's + * status between json values. + * + * ASYNC_PRESTART: We haven't seen any non-whitespace yet. We + * could be parsing an array, or not. We are waiting for valid + * JSON. + * + * ASYNC_START: We've seen an array and have begun unwrapping + * it. We could see a ] if the array is empty, or valid JSON. + * + * ASYNC_END: We've parsed an array and seen the final ]. At this + * point we should only see whitespace or an EOF. + * + * ASYNC_POSTVAL: We just parsed a value from inside the array. We + * expect to see whitespace, a comma, or a ]. + * + * ASYNC_PREVAL: We are in an array and we just saw a comma. We + * expect to see whitespace or a JSON value. + */ + @inline private[this] final def ASYNC_PRESTART = -5 + @inline private[this] final def ASYNC_START = -4 + @inline private[this] final def ASYNC_END = -3 + @inline private[this] final def ASYNC_POSTVAL = -2 + @inline private[this] final def ASYNC_PREVAL = -1 + + protected[jawn] def churn()(implicit facade: Facade[J]): Either[ParseException, Seq[J]] = { + + // accumulates json values + val results = mutable.ArrayBuffer.empty[J] + + // we rely on exceptions to tell us when we run out of data + try { + while (true) { + if (state < 0) { + (at(offset): @switch) match { + case '\n' => + newline(offset) + offset += 1 + + case ' ' | '\t' | '\r' => + offset += 1 + + case '[' => + if (state == ASYNC_PRESTART) { + offset += 1 + state = ASYNC_START + } else if (state == ASYNC_END) { + die(offset, "expected eof") + } else if (state == ASYNC_POSTVAL) { + die(offset, "expected , or ]") + } else { + state = 0 + } + + case ',' => + if (state == ASYNC_POSTVAL) { + offset += 1 + state = ASYNC_PREVAL + } else if (state == ASYNC_END) { + die(offset, "expected eof") + } else { + die(offset, "expected json value") + } + + case ']' => + if (state == ASYNC_POSTVAL || state == ASYNC_START) { + if (streamMode > 0) { + offset += 1 + state = ASYNC_END + } else { + die(offset, "expected json value or eof") + } + } else if (state == ASYNC_END) { + die(offset, "expected eof") + } else { + die(offset, "expected json value") + } + + case c => + if (state == ASYNC_END) { + die(offset, "expected eof") + } else if (state == ASYNC_POSTVAL) { + die(offset, "expected ] or ,") + } else { + if (state == ASYNC_PRESTART && streamMode > 0) streamMode = -1 + state = 0 + } + } + + } else { + // jump straight back into rparse + offset = reset(offset) + val (value, j) = if (state <= 0) { + parse(offset) + } else { + rparse(state, curr, stack) + } + if (streamMode > 0) { + state = ASYNC_POSTVAL + } else if (streamMode == 0) { + state = ASYNC_PREVAL + } else { + state = ASYNC_END + } + curr = j + offset = j + stack = Nil + results.append(value) + } + } + Right(results) + } catch { + case e: AsyncException => + if (done) { + // if we are done, make sure we ended at a good stopping point + if (state == ASYNC_PREVAL || state == ASYNC_END) Right(results) + else Left(ParseException("exhausted input", -1, -1, -1)) + } else { + // we ran out of data, so return what we have so far + Right(results) + } + + case e: ParseException => + // we hit a parser error, so return that error and results so far + Left(e) + } + } + + // every 1M we shift our array back by 1M. + protected[this] final def reset(i: Int): Int = { + if (offset >= 1048576) { + len -= 1048576 + offset -= 1048576 + pos -= 1048576 + System.arraycopy(data, 1048576, data, 0, len) + i - 1048576 + } else { + i + } + } + + /** + * We use this to keep track of the last recoverable place we've + * seen. If we hit an AsyncException, we can later resume from this + * point. + * + * This method is called during every loop of rparse, and the + * arguments are the exact arguments we can pass to rparse to + * continue where we left off. + */ + protected[this] final def checkpoint(state: Int, i: Int, stack: List[FContext[J]]) { + this.state = state + this.curr = i + this.stack = stack + } + + /** + * This is a specialized accessor for the case where our underlying data are + * bytes not chars. + */ + protected[this] final def byte(i: Int): Byte = + if (i >= len) throw new AsyncException else data(i) + + // we need to signal if we got out-of-bounds + protected[this] final def at(i: Int): Char = + if (i >= len) throw new AsyncException else data(i).toChar + + /** + * Access a byte range as a string. + * + * Since the underlying data are UTF-8 encoded, i and k must occur on unicode + * boundaries. Also, the resulting String is not guaranteed to have length + * (k - i). + */ + protected[this] final def at(i: Int, k: Int): CharSequence = { + if (k > len) throw new AsyncException + val size = k - i + val arr = new Array[Byte](size) + System.arraycopy(data, i, arr, 0, size) + new String(arr, utf8) + } + + // the basic idea is that we don't signal EOF until done is true, which means + // the client explicitly send us an EOF. + protected[this] final def atEof(i: Int): Boolean = + if (done) i >= len else false + + // we don't have to do anything special on close. + protected[this] final def close(): Unit = () +} + +/** + * This class is used internally by AsyncParser to signal that we've + * reached the end of the particular input we were given. + */ +private[jawn] class AsyncException extends Exception with control.NoStackTrace + +/** + * This is a more prosaic exception which indicates that we've hit a + * parsing error. + */ +private[jawn] class FailureException extends Exception diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/ByteBasedParser.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/ByteBasedParser.scala new file mode 100644 index 00000000..9fc5234a --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/ByteBasedParser.scala @@ -0,0 +1,104 @@ +package jawn + +import scala.annotation.{switch, tailrec} + +/** + * Trait used when the data to be parsed is in UTF-8. + * + * This parser has to translate input bytes to Chars and Strings. It + * provides a byte() method to access individual bytes, and also + * parser strings from bytes. + * + * Its parseString() implementation has two cases. In the first case + * (the hot path) the string has no escape sequences and we can just + * UTF-8 decode the entire set of bytes. In the second case, it goes + * to some trouble to be sure to de-escape correctly given that the + * input data is UTF-8. + */ +trait ByteBasedParser[J] extends Parser[J] { + protected[this] def byte(i: Int): Byte + + /** + * See if the string has any escape sequences. If not, return the end of the + * string. If so, bail out and return -1. + * + * This method expects the data to be in UTF-8 and accesses it as bytes. Thus + * we can just ignore any bytes with the highest bit set. + */ + protected[this] final def parseStringSimple(i: Int, ctxt: FContext[J]): Int = { + var j = i + var c: Int = byte(j) & 0xff + while (c != 34) { + if (c < 32) return die(j, s"control char ($c) in string") + if (c == 92) return -1 + j += 1 + c = byte(j) & 0xff + } + j + 1 + } + + /** + * Parse the string according to JSON rules, and add to the given context. + * + * This method expects the data to be in UTF-8 and accesses it as bytes. + */ + protected[this] final def parseString(i: Int, ctxt: FContext[J]): Int = { + val k = parseStringSimple(i + 1, ctxt) + if (k != -1) { + ctxt.add(at(i + 1, k - 1)) + return k + } + + // TODO: we might be able to do better by identifying where + // escapes occur, and then translating the intermediate strings in + // one go. + + var j = i + 1 + val sb = new CharBuilder + + var c: Int = byte(j) & 0xff + while (c != 34) { // " + if (c == 92) { // \ + (byte(j + 1): @switch) match { + case 98 => { sb.append('\b'); j += 2 } + case 102 => { sb.append('\f'); j += 2 } + case 110 => { sb.append('\n'); j += 2 } + case 114 => { sb.append('\r'); j += 2 } + case 116 => { sb.append('\t'); j += 2 } + + case 34 => { sb.append('"'); j += 2 } + case 47 => { sb.append('/'); j += 2 } + case 92 => { sb.append('\\'); j += 2 } + + // if there's a problem then descape will explode + case 117 => { sb.append(descape(at(j + 2, j + 6))); j += 6 } + + case c => die(j, s"invalid escape sequence (\\${c.toChar})") + } + } else if (c < 32) { + die(j, s"control char ($c) in string") + } else if (c < 128) { + // 1-byte UTF-8 sequence + sb.append(c.toChar) + j += 1 + } else if ((c & 224) == 192) { + // 2-byte UTF-8 sequence + sb.extend(at(j, j + 2)) + j += 2 + } else if ((c & 240) == 224) { + // 3-byte UTF-8 sequence + sb.extend(at(j, j + 3)) + j += 3 + } else if ((c & 248) == 240) { + // 4-byte UTF-8 sequence + sb.extend(at(j, j + 4)) + j += 4 + } else { + die(j, "invalid UTF-8 encoding") + } + c = byte(j) & 0xff + } + ctxt.add(sb.makeString) + j + 1 + } +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/ByteBufferParser.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/ByteBufferParser.scala new file mode 100644 index 00000000..1902b8d2 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/ByteBufferParser.scala @@ -0,0 +1,42 @@ +package jawn + +import scala.annotation.{switch, tailrec} +import java.nio.ByteBuffer + +/** + * Basic ByteBuffer parser. + * + * This assumes that the provided ByteBuffer is ready to be read. The + * user is responsible for any necessary flipping/resetting of the + * ByteBuffer before parsing. + * + * The parser makes absolute calls to the ByteBuffer, which will not + * update its own mutable position fields. + */ +final class ByteBufferParser[J](src: ByteBuffer) extends SyncParser[J] with ByteBasedParser[J] { + private[this] final val start = src.position + private[this] final val limit = src.limit - start + + private[this] var lineState = 0 + protected[this] def line(): Int = lineState + + protected[this] final def newline(i: Int) { lineState += 1 } + protected[this] final def column(i: Int) = i + + protected[this] final def close() { src.position(src.limit) } + protected[this] final def reset(i: Int): Int = i + protected[this] final def checkpoint(state: Int, i: Int, stack: List[FContext[J]]) {} + protected[this] final def byte(i: Int): Byte = src.get(i + start) + protected[this] final def at(i: Int): Char = src.get(i + start).toChar + + protected[this] final def at(i: Int, k: Int): CharSequence = { + val len = k - i + val arr = new Array[Byte](len) + src.position(i + start) + src.get(arr, 0, len) + src.position(start) + new String(arr, utf8) + } + + protected[this] final def atEof(i: Int) = i >= limit +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/ChannelParser.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/ChannelParser.scala new file mode 100644 index 00000000..3c93e741 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/ChannelParser.scala @@ -0,0 +1,164 @@ +package jawn + +import java.lang.Integer.{ bitCount, highestOneBit } +import java.io.{File, FileInputStream} +import java.nio.ByteBuffer +import java.nio.channels.ReadableByteChannel + +object ChannelParser { + + final val DefaultBufferSize = 1048576 + + final val ParseAsStringThreshold = 20 * 1048576 + + def fromFile[J](f: File, bufferSize: Int = DefaultBufferSize): SyncParser[J] = + if (f.length < ParseAsStringThreshold) { + val bytes = new Array[Byte](f.length.toInt) + val fis = new FileInputStream(f) + fis.read(bytes) + new StringParser[J](new String(bytes, "UTF-8")) + } else { + new ChannelParser[J](new FileInputStream(f).getChannel, bufferSize) + } + + def fromChannel[J](ch: ReadableByteChannel, bufferSize: Int = DefaultBufferSize): ChannelParser[J] = + new ChannelParser[J](ch, bufferSize) + + /** + * Given a desired buffer size, find the closest positive + * power-of-two larger than that size. + * + * This method throws an exception if the given values are negative + * or too large to have a valid power of two. + */ + def computeBufferSize(x: Int): Int = + if (x < 0) { + throw new IllegalArgumentException("negative bufferSize ($x)") + } else if (x > 0x40000000) { + throw new IllegalArgumentException("bufferSize too large ($x)") + } else if (bitCount(x) == 1) { + x + } else { + highestOneBit(x) << 1 + } +} + +/** + * Basic file parser. + * + * Given a file name this parser opens it, chunks the data, and parses + * it. + */ +final class ChannelParser[J](ch: ReadableByteChannel, bufferSize: Int) extends SyncParser[J] with ByteBasedParser[J] { + + var Bufsize: Int = ChannelParser.computeBufferSize(bufferSize) + var Mask: Int = Bufsize - 1 + var Allsize: Int = Bufsize * 2 + + // these are the actual byte arrays we'll use + private var curr = new Array[Byte](Bufsize) + private var next = new Array[Byte](Bufsize) + + // these are the bytecounts for each array + private var ncurr = ch.read(ByteBuffer.wrap(curr)) + private var nnext = ch.read(ByteBuffer.wrap(next)) + + var line = 0 + private var pos = 0 + protected[this] final def newline(i: Int): Unit = { line += 1; pos = i } + protected[this] final def column(i: Int): Int = i - pos + + protected[this] final def close(): Unit = ch.close() + + /** + * Swap the curr and next arrays/buffers/counts. + * + * We'll call this in response to certain reset() calls. Specifically, when + * the index provided to reset is no longer in the 'curr' buffer, we want to + * clear that data and swap the buffers. + */ + protected[this] final def swap(): Unit = { + var tmp = curr; curr = next; next = tmp + var ntmp = ncurr; ncurr = nnext; nnext = ntmp + } + + protected[this] final def grow(): Unit = { + val cc = new Array[Byte](Allsize) + System.arraycopy(curr, 0, cc, 0, Bufsize) + System.arraycopy(next, 0, cc, Bufsize, Bufsize) + + curr = cc + ncurr = ncurr + nnext + next = new Array[Byte](Allsize) + nnext = ch.read(ByteBuffer.wrap(next)) + + Bufsize = Allsize + Mask = Allsize - 1 + Allsize *= 2 + } + + /** + * If the cursor 'i' is past the 'curr' buffer, we want to clear the + * current byte buffer, do a swap, load some more data, and + * continue. + */ + protected[this] final def reset(i: Int): Int = + if (i >= Bufsize) { + swap() + nnext = ch.read(ByteBuffer.wrap(next)) + pos -= Bufsize + i - Bufsize + } else { + i + } + + protected[this] final def checkpoint(state: Int, i: Int, stack: List[FContext[J]]): Unit = () + + /** + * This is a specialized accessor for the case where our underlying + * data are bytes not chars. + */ + protected[this] final def byte(i: Int): Byte = + if (i < Bufsize) curr(i) + else if (i < Allsize) next(i & Mask) + else { grow(); byte(i) } + + /** + * Reads a byte as a single Char. The byte must be valid ASCII (this + * method is used to parse JSON values like numbers, constants, or + * delimiters, which are known to be within ASCII). + */ + protected[this] final def at(i: Int): Char = + if (i < Bufsize) curr(i).toChar + else if (i < Allsize) next(i & Mask).toChar + else { grow(); at(i) } + + /** + * Access a byte range as a string. + * + * Since the underlying data are UTF-8 encoded, i and k must occur + * on unicode boundaries. Also, the resulting String is not + * guaranteed to have length (k - i). + */ + protected[this] final def at(i: Int, k: Int): CharSequence = { + val len = k - i + if (k > Allsize) { + grow() + at(i, k) + } else if (k <= Bufsize) { + new String(curr, i, len, utf8) + } else if (i >= Bufsize) { + new String(next, i - Bufsize, len, utf8) + } else { + val arr = new Array[Byte](len) + val mid = Bufsize - i + System.arraycopy(curr, i, arr, 0, mid) + System.arraycopy(next, 0, arr, mid, k - Bufsize) + new String(arr, utf8) + } + } + + protected[this] final def atEof(i: Int) = + if (i < Bufsize) i >= ncurr + else i >= (nnext + Bufsize) +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/CharBasedParser.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/CharBasedParser.scala new file mode 100644 index 00000000..a054e5dc --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/CharBasedParser.scala @@ -0,0 +1,98 @@ +package jawn + +import scala.annotation.{switch, tailrec} + +/** + * Trait used when the data to be parsed is in UTF-16. + * + * This parser provides parseString(). Like ByteBasedParser it has + * fast/slow paths for string parsing depending on whether any escapes + * are present. + * + * It is simpler than ByteBasedParser. + */ +trait CharBasedParser[J] extends Parser[J] { + + private[this] final val charBuilder = new CharBuilder() + + /** + * See if the string has any escape sequences. If not, return the + * end of the string. If so, bail out and return -1. + * + * This method expects the data to be in UTF-16 and accesses it as + * chars. + */ + protected[this] final def parseStringSimple(i: Int, ctxt: FContext[J]): Int = { + var j = i + var c = at(j) + while (c != '"') { + if (c < ' ') return die(j, s"control char (${c.toInt}) in string") + if (c == '\\') return -1 + j += 1 + c = at(j) + } + j + 1 + } + + /** + * Parse a string that is known to have escape sequences. + */ + protected[this] final def parseStringComplex(i: Int, ctxt: FContext[J]): Int = { + var j = i + 1 + val sb = charBuilder.reset() + + var c = at(j) + while (c != '"') { + if (c < ' ') { + die(j, s"control char (${c.toInt}) in string") + } else if (c == '\\') { + (at(j + 1): @switch) match { + case 'b' => { sb.append('\b'); j += 2 } + case 'f' => { sb.append('\f'); j += 2 } + case 'n' => { sb.append('\n'); j += 2 } + case 'r' => { sb.append('\r'); j += 2 } + case 't' => { sb.append('\t'); j += 2 } + + case '"' => { sb.append('"'); j += 2 } + case '/' => { sb.append('/'); j += 2 } + case '\\' => { sb.append('\\'); j += 2 } + + // if there's a problem then descape will explode + case 'u' => { sb.append(descape(at(j + 2, j + 6))); j += 6 } + + case c => die(j, s"illegal escape sequence (\\$c)") + } + } else { + // this case is for "normal" code points that are just one Char. + // + // we don't have to worry about surrogate pairs, since those + // will all be in the ranges D800–DBFF (high surrogates) or + // DC00–DFFF (low surrogates). + sb.append(c) + j += 1 + } + j = reset(j) + c = at(j) + } + ctxt.add(sb.makeString) + j + 1 + } + + /** + * Parse the string according to JSON rules, and add to the given + * context. + * + * This method expects the data to be in UTF-16, and access it as + * Char. It performs the correct checks to make sure that we don't + * interpret a multi-char code point incorrectly. + */ + protected[this] final def parseString(i: Int, ctxt: FContext[J]): Int = { + val k = parseStringSimple(i + 1, ctxt) + if (k != -1) { + ctxt.add(at(i + 1, k - 1)) + k + } else { + parseStringComplex(i, ctxt) + } + } +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/CharBuilder.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/CharBuilder.scala new file mode 100644 index 00000000..589437bf --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/CharBuilder.scala @@ -0,0 +1,56 @@ +package jawn + +/** + * CharBuilder is a specialized way to build Strings. + * + * It wraps a (growable) array of characters, and can accept + * additional String or Char data to be added to its buffer. + */ +private[jawn] final class CharBuilder { + @inline final def INITIALSIZE = 32 + + private var cs = new Array[Char](INITIALSIZE) + private var capacity = INITIALSIZE + private var len = 0 + + def reset(): CharBuilder = { + len = 0 + this + } + + def makeString: String = new String(cs, 0, len) + + def resizeIfNecessary(goal: Int): Unit = { + if (goal <= capacity) return () + var cap = capacity + while (goal > cap && cap > 0) cap *= 2 + if (cap > capacity) { + val ncs = new Array[Char](cap) + System.arraycopy(cs, 0, ncs, 0, capacity) + cs = ncs + capacity = cap + } else if (cap < capacity) { + sys.error("maximum string size exceeded") + } + } + + def extend(s: CharSequence): Unit = { + val tlen = len + s.length + resizeIfNecessary(tlen) + var i = 0 + var j = len + len = tlen + while (i < s.length) { + cs(j) = s.charAt(i) + i += 1 + j += 1 + } + } + + def append(c: Char): Unit = { + val tlen = len + 1 + resizeIfNecessary(tlen) + cs(len) = c + len = tlen + } +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/CharSequenceParser.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/CharSequenceParser.scala new file mode 100644 index 00000000..c592326e --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/CharSequenceParser.scala @@ -0,0 +1,18 @@ +package jawn + +/** + * Lazy character sequence parsing. + * + * This is similar to StringParser, but acts on character sequences. + */ +private[jawn] final class CharSequenceParser[J](cs: CharSequence) extends SyncParser[J] with CharBasedParser[J] { + var line = 0 + final def column(i: Int) = i + final def newline(i: Int) { line += 1 } + final def reset(i: Int): Int = i + final def checkpoint(state: Int, i: Int, stack: List[FContext[J]]): Unit = () + final def at(i: Int): Char = cs.charAt(i) + final def at(i: Int, j: Int): CharSequence = cs.subSequence(i, j) + final def atEof(i: Int) = i == cs.length + final def close() = () +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/Facade.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/Facade.scala new file mode 100644 index 00000000..203b68e9 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/Facade.scala @@ -0,0 +1,34 @@ +package jawn + +/** + * Facade is a type class that describes how Jawn should construct + * JSON AST elements of type J. + * + * Facade[J] also uses FContext[J] instances, so implementors will + * usually want to define both. + */ +trait Facade[J] { + def singleContext(): FContext[J] + def arrayContext(): FContext[J] + def objectContext(): FContext[J] + + def jnull(): J + def jfalse(): J + def jtrue(): J + def jnum(s: CharSequence, decIndex: Int, expIndex: Int): J + def jstring(s: CharSequence): J +} + +/** + * FContext is used to construct nested JSON values. + * + * The most common cases are to build objects and arrays. However, + * this type is also used to build a single top-level JSON element, in + * cases where the entire JSON document consists of "333.33". + */ +trait FContext[J] { + def add(s: CharSequence): Unit + def add(v: J): Unit + def finish: J + def isObj: Boolean +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/MutableFacade.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/MutableFacade.scala new file mode 100644 index 00000000..8fe5716b --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/MutableFacade.scala @@ -0,0 +1,35 @@ +package jawn + +import scala.collection.mutable + +trait MutableFacade[J] extends Facade[J] { + def jarray(vs: mutable.ArrayBuffer[J]): J + def jobject(vs: mutable.Map[String, J]): J + + def singleContext() = new FContext[J] { + var value: J = _ + def add(s: CharSequence) { value = jstring(s) } + def add(v: J) { value = v } + def finish: J = value + def isObj: Boolean = false + } + + def arrayContext() = new FContext[J] { + val vs = mutable.ArrayBuffer.empty[J] + def add(s: CharSequence) { vs.append(jstring(s)) } + def add(v: J) { vs.append(v) } + def finish: J = jarray(vs) + def isObj: Boolean = false + } + + def objectContext() = new FContext[J] { + var key: String = null + val vs = mutable.Map.empty[String, J] + def add(s: CharSequence): Unit = + if (key == null) { key = s.toString } else { vs(key) = jstring(s); key = null } + def add(v: J): Unit = + { vs(key) = v; key = null } + def finish = jobject(vs) + def isObj = true + } +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/NullFacade.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/NullFacade.scala new file mode 100644 index 00000000..39d55884 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/NullFacade.scala @@ -0,0 +1,30 @@ +package jawn + +/** + * NullFacade discards all JSON AST information. + * + * This is the simplest possible facade. It could be useful for + * checking JSON for correctness (via parsing) without worrying about + * saving the data. + * + * It will always return () on any successful parse, no matter the + * content. + */ +object NullFacade extends Facade[Unit] { + + case class NullContext(isObj: Boolean) extends FContext[Unit] { + def add(s: CharSequence): Unit = () + def add(v: Unit): Unit = () + def finish: Unit = () + } + + val singleContext: FContext[Unit] = NullContext(false) + val arrayContext: FContext[Unit] = NullContext(false) + val objectContext: FContext[Unit] = NullContext(true) + + def jnull(): Unit = () + def jfalse(): Unit = () + def jtrue(): Unit = () + def jnum(s: CharSequence, decIndex: Int, expIndex: Int): Unit = () + def jstring(s: CharSequence): Unit = () +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/Parser.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/Parser.scala new file mode 100644 index 00000000..1177e91f --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/Parser.scala @@ -0,0 +1,507 @@ +package jawn + +import java.io.File +import java.nio.ByteBuffer +import java.nio.channels.ReadableByteChannel +import java.nio.charset.Charset +import scala.annotation.{switch, tailrec} +import scala.util.Try + +case class ParseException(msg: String, index: Int, line: Int, col: Int) extends Exception(msg) + +case class IncompleteParseException(msg: String) extends Exception(msg) + +/** + * Parser implements a state machine for correctly parsing JSON data. + * + * The trait relies on a small number of methods which are left + * abstract, and which generalize parsing based on whether the input + * is in Bytes or Chars, coming from Strings, files, or other input. + * All methods provided here are protected, so different parsers can + * choose which functionality to expose. + * + * Parser is parameterized on J, which is the type of the JSON AST it + * will return. Jawn can produce any AST for which a Facade[J] is + * available. + * + * The parser trait does not hold any state itself, but particular + * implementations will usually hold state. Parser instances should + * not be reused between parsing runs. + * + * For now the parser requires input to be in UTF-8. This requirement + * may eventually be relaxed. + */ +abstract class Parser[J] { + + protected[this] final val utf8 = Charset.forName("UTF-8") + + /** + * Read the byte/char at 'i' as a Char. + * + * Note that this should not be used on potential multi-byte + * sequences. + */ + protected[this] def at(i: Int): Char + + /** + * Read the bytes/chars from 'i' until 'j' as a String. + */ + protected[this] def at(i: Int, j: Int): CharSequence + + /** + * Return true iff 'i' is at or beyond the end of the input (EOF). + */ + protected[this] def atEof(i: Int): Boolean + + /** + * The reset() method is used to signal that we're working from the + * given position, and any previous data can be released. Some + * parsers (e.g. StringParser) will ignore release, while others + * (e.g. PathParser) will need to use this information to release + * and allocate different areas. + */ + protected[this] def reset(i: Int): Int + + /** + * The checkpoint() method is used to allow some parsers to store + * their progress. + */ + protected[this] def checkpoint(state: Int, i: Int, stack: List[FContext[J]]): Unit + + /** + * Should be called when parsing is finished. + */ + protected[this] def close(): Unit + + /** + * Valid parser states. + */ + @inline protected[this] final val ARRBEG = 6 + @inline protected[this] final val OBJBEG = 7 + @inline protected[this] final val DATA = 1 + @inline protected[this] final val KEY = 2 + @inline protected[this] final val SEP = 3 + @inline protected[this] final val ARREND = 4 + @inline protected[this] final val OBJEND = 5 + + protected[this] def newline(i: Int): Unit + protected[this] def line(): Int + protected[this] def column(i: Int): Int + + protected[this] final val HexChars: Array[Int] = { + val arr = new Array[Int](128) + var i = 0 + while (i < 10) { arr(i + '0') = i; i += 1 } + i = 0 + while (i < 16) { arr(i + 'a') = 10 + i; arr(i + 'A') = 10 + i; i += 1 } + arr + } + + /** + * Used to generate error messages with character info and offsets. + */ + protected[this] def die(i: Int, msg: String): Nothing = { + val y = line() + 1 + val x = column(i) + 1 + val s = "%s got %s (line %d, column %d)" format (msg, at(i), y, x) + throw ParseException(s, i, y, x) + } + + /** + * Used to generate messages for internal errors. + * + * This should only be used in situations where a possible bug in + * the parser was detected. For errors in user-provided JSON, use + * die(). + */ + protected[this] def error(msg: String) = + sys.error(msg) + + /** + * Parse the given number, and add it to the given context. + * + * We don't actually instantiate a number here, but rather pass the + * string of for future use. Facades can choose to be lazy and just + * store the string. This ends up being way faster and has the nice + * side-effect that we know exactly how the user represented the + * number. + */ + protected[this] final def parseNum(i: Int, ctxt: FContext[J])(implicit facade: Facade[J]): Int = { + var j = i + var c = at(j) + var decIndex = -1 + var expIndex = -1 + + if (c == '-') { + j += 1 + c = at(j) + } + if (c == '0') { + j += 1 + c = at(j) + } else if ('1' <= c && c <= '9') { + while ('0' <= c && c <= '9') { j += 1; c = at(j) } + } else { + die(i, "expected digit") + } + + if (c == '.') { + decIndex = j - i + j += 1 + c = at(j) + if ('0' <= c && c <= '9') { + while ('0' <= c && c <= '9') { j += 1; c = at(j) } + } else { + die(i, "expected digit") + } + } + + if (c == 'e' || c == 'E') { + expIndex = j - i + j += 1 + c = at(j) + if (c == '+' || c == '-') { + j += 1 + c = at(j) + } + if ('0' <= c && c <= '9') { + while ('0' <= c && c <= '9') { j += 1; c = at(j) } + } else { + die(i, "expected digit") + } + } + + ctxt.add(facade.jnum(at(i, j), decIndex, expIndex)) + j + } + + /** + * Parse the given number, and add it to the given context. + * + * This method is a bit slower than parseNum() because it has to be + * sure it doesn't run off the end of the input. + * + * Normally (when operating in rparse in the context of an outer + * array or object) we don't need to worry about this and can just + * grab characters, because if we run out of characters that would + * indicate bad input. This is for cases where the number could + * possibly be followed by a valid EOF. + * + * This method has all the same caveats as the previous method. + */ + protected[this] final def parseNumSlow(i: Int, ctxt: FContext[J])(implicit facade: Facade[J]): Int = { + var j = i + var c = at(j) + var decIndex = -1 + var expIndex = -1 + + if (c == '-') { + // any valid input will require at least one digit after - + j += 1 + c = at(j) + } + if (c == '0') { + j += 1 + if (atEof(j)) { + ctxt.add(facade.jnum(at(i, j), decIndex, expIndex)) + return j + } + c = at(j) + } else if ('1' <= c && c <= '9') { + while ('0' <= c && c <= '9') { + j += 1 + if (atEof(j)) { + ctxt.add(facade.jnum(at(i, j), decIndex, expIndex)) + return j + } + c = at(j) + } + } else { + die(i, "expected digit") + } + + if (c == '.') { + // any valid input will require at least one digit after . + decIndex = j - i + j += 1 + c = at(j) + if ('0' <= c && c <= '9') { + while ('0' <= c && c <= '9') { + j += 1 + if (atEof(j)) { + ctxt.add(facade.jnum(at(i, j), decIndex, expIndex)) + return j + } + c = at(j) + } + } else { + die(i, "expected digit") + } + } + + if (c == 'e' || c == 'E') { + // any valid input will require at least one digit after e, e+, etc + expIndex = j - i + j += 1 + c = at(j) + if (c == '+' || c == '-') { + j += 1 + c = at(j) + } + if ('0' <= c && c <= '9') { + while ('0' <= c && c <= '9') { + j += 1 + if (atEof(j)) { + ctxt.add(facade.jnum(at(i, j), decIndex, expIndex)) + return j + } + c = at(j) + } + } else { + die(i, "expected digit") + } + } + + ctxt.add(facade.jnum(at(i, j), decIndex, expIndex)) + j + } + + /** + * Generate a Char from the hex digits of "\u1234" (i.e. "1234"). + * + * NOTE: This is only capable of generating characters from the basic plane. + * This is why it can only return Char instead of Int. + */ + protected[this] final def descape(s: CharSequence): Char = { + val hc = HexChars + var i = 0 + var x = 0 + while (i < 4) { + x = (x << 4) | hc(s.charAt(i).toInt) + i += 1 + } + x.toChar + } + + /** + * Parse the JSON string starting at 'i' and save it into 'ctxt'. + */ + protected[this] def parseString(i: Int, ctxt: FContext[J]): Int + + /** + * Parse the JSON constant "true". + * + * Note that this method assumes that the first character has already been checked. + */ + protected[this] final def parseTrue(i: Int)(implicit facade: Facade[J]): J = + if (at(i + 1) == 'r' && at(i + 2) == 'u' && at(i + 3) == 'e') { + facade.jtrue + } else { + die(i, "expected true") + } + + /** + * Parse the JSON constant "false". + * + * Note that this method assumes that the first character has already been checked. + */ + protected[this] final def parseFalse(i: Int)(implicit facade: Facade[J]): J = + if (at(i + 1) == 'a' && at(i + 2) == 'l' && at(i + 3) == 's' && at(i + 4) == 'e') { + facade.jfalse + } else { + die(i, "expected false") + } + + /** + * Parse the JSON constant "null". + * + * Note that this method assumes that the first character has already been checked. + */ + protected[this] final def parseNull(i: Int)(implicit facade: Facade[J]): J = + if (at(i + 1) == 'u' && at(i + 2) == 'l' && at(i + 3) == 'l') { + facade.jnull + } else { + die(i, "expected null") + } + + /** + * Parse and return the next JSON value and the position beyond it. + */ + protected[this] final def parse(i: Int)(implicit facade: Facade[J]): (J, Int) = try { + (at(i): @switch) match { + // ignore whitespace + case ' ' => parse(i + 1) + case '\t' => parse(i + 1) + case '\r' => parse(i + 1) + case '\n' => newline(i); parse(i + 1) + + // if we have a recursive top-level structure, we'll delegate the parsing + // duties to our good friend rparse(). + case '[' => rparse(ARRBEG, i + 1, facade.arrayContext() :: Nil) + case '{' => rparse(OBJBEG, i + 1, facade.objectContext() :: Nil) + + // we have a single top-level number + case '-' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' => + val ctxt = facade.singleContext() + val j = parseNumSlow(i, ctxt) + (ctxt.finish, j) + + // we have a single top-level string + case '"' => + val ctxt = facade.singleContext() + val j = parseString(i, ctxt) + (ctxt.finish, j) + + // we have a single top-level constant + case 't' => (parseTrue(i), i + 4) + case 'f' => (parseFalse(i), i + 5) + case 'n' => (parseNull(i), i + 4) + + // invalid + case _ => die(i, "expected json value") + } + } catch { + case _: IndexOutOfBoundsException => + throw IncompleteParseException("exhausted input") + } + + /** + * Tail-recursive parsing method to do the bulk of JSON parsing. + * + * This single method manages parser states, data, etc. Except for + * parsing non-recursive values (like strings, numbers, and + * constants) all important work happens in this loop (or in methods + * it calls, like reset()). + * + * Currently the code is optimized to make use of switch + * statements. Future work should consider whether this is better or + * worse than manually constructed if/else statements or something + * else. Also, it may be possible to reorder some cases for speed + * improvements. + */ + @tailrec + protected[this] final def rparse(state: Int, j: Int, stack: List[FContext[J]])(implicit facade: Facade[J]): (J, Int) = { + val i = reset(j) + checkpoint(state, i, stack) + + val c = at(i) + + if (c == '\n') { + newline(i) + rparse(state, i + 1, stack) + } else if (c == ' ' || c == '\t' || c == '\r') { + rparse(state, i + 1, stack) + } else if (state == DATA) { + // we are inside an object or array expecting to see data + if (c == '[') { + rparse(ARRBEG, i + 1, facade.arrayContext() :: stack) + } else if (c == '{') { + rparse(OBJBEG, i + 1, facade.objectContext() :: stack) + } else { + val ctxt = stack.head + + if ((c >= '0' && c <= '9') || c == '-') { + val j = parseNum(i, ctxt) + rparse(if (ctxt.isObj) OBJEND else ARREND, j, stack) + } else if (c == '"') { + val j = parseString(i, ctxt) + rparse(if (ctxt.isObj) OBJEND else ARREND, j, stack) + } else if (c == 't') { + ctxt.add(parseTrue(i)) + rparse(if (ctxt.isObj) OBJEND else ARREND, i + 4, stack) + } else if (c == 'f') { + ctxt.add(parseFalse(i)) + rparse(if (ctxt.isObj) OBJEND else ARREND, i + 5, stack) + } else if (c == 'n') { + ctxt.add(parseNull(i)) + rparse(if (ctxt.isObj) OBJEND else ARREND, i + 4, stack) + } else { + die(i, "expected json value") + } + } + } else if ( + (c == ']' && (state == ARREND || state == ARRBEG)) || + (c == '}' && (state == OBJEND || state == OBJBEG)) + ) { + // we are inside an array or object and have seen a key or a closing + // brace, respectively. + if (stack.isEmpty) { + error("invalid stack") + } else { + val ctxt1 = stack.head + val tail = stack.tail + + if (tail.isEmpty) { + (ctxt1.finish, i + 1) + } else { + val ctxt2 = tail.head + ctxt2.add(ctxt1.finish) + rparse(if (ctxt2.isObj) OBJEND else ARREND, i + 1, tail) + } + } + } else if (state == KEY) { + // we are in an object expecting to see a key. + if (c == '"') { + val j = parseString(i, stack.head) + rparse(SEP, j, stack) + } else { + die(i, "expected \"") + } + } else if (state == SEP) { + // we are in an object just after a key, expecting to see a colon. + if (c == ':') { + rparse(DATA, i + 1, stack) + } else { + die(i, "expected :") + } + } else if (state == ARREND) { + // we are in an array, expecting to see a comma (before more data). + if (c == ',') { + rparse(DATA, i + 1, stack) + } else { + die(i, "expected ] or ,") + } + } else if (state == OBJEND) { + // we are in an object, expecting to see a comma (before more data). + if (c == ',') { + rparse(KEY, i + 1, stack) + } else { + die(i, "expected } or ,") + } + } else if (state == ARRBEG) { + // we are starting an array, expecting to see data or a closing bracket. + rparse(DATA, i, stack) + } else { + // we are starting an object, expecting to see a key or a closing brace. + rparse(KEY, i, stack) + } + } +} + + +object Parser { + + def parseUnsafe[J](s: String)(implicit facade: Facade[J]): J = + new StringParser(s).parse() + + def parseFromString[J](s: String)(implicit facade: Facade[J]): Try[J] = + Try(new StringParser[J](s).parse) + + def parseFromCharSequence[J](cs: CharSequence)(implicit facade: Facade[J]): Try[J] = + Try(new CharSequenceParser[J](cs).parse) + + def parseFromPath[J](path: String)(implicit facade: Facade[J]): Try[J] = + Try(ChannelParser.fromFile[J](new File(path)).parse) + + def parseFromFile[J](file: File)(implicit facade: Facade[J]): Try[J] = + Try(ChannelParser.fromFile[J](file).parse) + + def parseFromChannel[J](ch: ReadableByteChannel)(implicit facade: Facade[J]): Try[J] = + Try(ChannelParser.fromChannel[J](ch).parse) + + def parseFromByteBuffer[J](buf: ByteBuffer)(implicit facade: Facade[J]): Try[J] = + Try(new ByteBufferParser[J](buf).parse) + + def async[J](mode: AsyncParser.Mode)(implicit facade: Facade[J]): AsyncParser[J] = + AsyncParser[J](mode) +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/SimpleFacade.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/SimpleFacade.scala new file mode 100644 index 00000000..dabec016 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/SimpleFacade.scala @@ -0,0 +1,42 @@ +package jawn + +import scala.collection.mutable + +/** + * Facade is a type class that describes how Jawn should construct + * JSON AST elements of type J. + * + * Facade[J] also uses FContext[J] instances, so implementors will + * usually want to define both. + */ +trait SimpleFacade[J] extends Facade[J] { + def jarray(vs: List[J]): J + def jobject(vs: Map[String, J]): J + + def singleContext() = new FContext[J] { + var value: J = _ + def add(s: CharSequence) { value = jstring(s) } + def add(v: J) { value = v } + def finish: J = value + def isObj: Boolean = false + } + + def arrayContext() = new FContext[J] { + val vs = mutable.ListBuffer.empty[J] + def add(s: CharSequence) { vs += jstring(s) } + def add(v: J) { vs += v } + def finish: J = jarray(vs.toList) + def isObj: Boolean = false + } + + def objectContext() = new FContext[J] { + var key: String = null + var vs = Map.empty[String, J] + def add(s: CharSequence): Unit = + if (key == null) { key = s.toString } else { vs = vs.updated(key, jstring(s)); key = null } + def add(v: J): Unit = + { vs = vs.updated(key, v); key = null } + def finish = jobject(vs) + def isObj = true + } +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/StringParser.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/StringParser.scala new file mode 100644 index 00000000..91662fc0 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/StringParser.scala @@ -0,0 +1,25 @@ +package jawn + +/** + * Basic in-memory string parsing. + * + * This is probably the simplest Parser implementation, since there is + * no UTF-8 decoding, and the data is already fully available. + * + * This parser is limited to the maximum string size (~2G). Obviously + * for large JSON documents it's better to avoid using this parser and + * go straight from disk, to avoid having to load the whole thing into + * memory at once. So this limit will probably not be a problem in + * practice. + */ +private[jawn] final class StringParser[J](s: String) extends SyncParser[J] with CharBasedParser[J] { + var line = 0 + final def column(i: Int) = i + final def newline(i: Int) { line += 1 } + final def reset(i: Int): Int = i + final def checkpoint(state: Int, i: Int, stack: List[FContext[J]]): Unit = () + final def at(i: Int): Char = s.charAt(i) + final def at(i: Int, j: Int): CharSequence = s.substring(i, j) + final def atEof(i: Int) = i == s.length + final def close() = () +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/SupportParser.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/SupportParser.scala new file mode 100644 index 00000000..2304a8dd --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/SupportParser.scala @@ -0,0 +1,31 @@ +package jawn + +import java.io.File +import java.nio.ByteBuffer +import java.nio.channels.ReadableByteChannel +import scala.util.Try + +trait SupportParser[J] { + implicit def facade: Facade[J] + + def parseUnsafe(s: String): J = + new StringParser(s).parse() + + def parseFromString(s: String): Try[J] = + Try(new StringParser[J](s).parse) + + def parseFromPath(path: String): Try[J] = + Try(ChannelParser.fromFile[J](new File(path)).parse) + + def parseFromFile(file: File): Try[J] = + Try(ChannelParser.fromFile[J](file).parse) + + def parseFromChannel(ch: ReadableByteChannel): Try[J] = + Try(ChannelParser.fromChannel[J](ch).parse) + + def parseFromByteBuffer(buf: ByteBuffer): Try[J] = + Try(new ByteBufferParser[J](buf).parse) + + def async(mode: AsyncParser.Mode): AsyncParser[J] = + AsyncParser[J](mode) +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/SyncParser.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/SyncParser.scala new file mode 100644 index 00000000..988a8ca9 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/SyncParser.scala @@ -0,0 +1,37 @@ +package jawn + +import scala.annotation.{switch, tailrec} +import scala.collection.mutable + +/** + * SyncParser extends Parser to do all parsing synchronously. + * + * Most traditional JSON parser are synchronous, and expect to receive + * all their input before returning. SyncParser[J] still leaves + * Parser[J]'s methods abstract, but adds a public methods for users + * to call to actually parse JSON. + */ +abstract class SyncParser[J] extends Parser[J] { + + /** + * Parse the JSON document into a single JSON value. + * + * The parser considers documents like '333', 'true', and '"foo"' to be + * valid, as well as more traditional documents like [1,2,3,4,5]. However, + * multiple top-level objects are not allowed. + */ + final def parse()(implicit facade: Facade[J]): J = { + val (value, i) = parse(0) + var j = i + while (!atEof(j)) { + (at(j): @switch) match { + case '\n' => newline(j); j += 1 + case ' ' | '\t' | '\r' => j += 1 + case _ => die(j, "expected whitespace or eof") + } + } + if (!atEof(j)) die(j, "expected eof") + close() + value + } +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/Syntax.scala b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/Syntax.scala new file mode 100644 index 00000000..119b5783 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/main/scala/jawn/Syntax.scala @@ -0,0 +1,27 @@ +package jawn + +import java.io.File +import java.nio.ByteBuffer +import java.nio.channels.ReadableByteChannel +import java.nio.charset.Charset +import scala.annotation.{switch, tailrec} +import scala.util.Try + +object Syntax { + implicit def unitFacade: Facade[Unit] = NullFacade + + def checkString(s: String): Boolean = + Try(new StringParser(s).parse).isSuccess + + def checkPath(path: String): Boolean = + Try(ChannelParser.fromFile(new File(path)).parse).isSuccess + + def checkFile(file: File): Boolean = + Try(ChannelParser.fromFile(file).parse).isSuccess + + def checkChannel(ch: ReadableByteChannel): Boolean = + Try(ChannelParser.fromChannel(ch).parse).isSuccess + + def checkByteBuffer(buf: ByteBuffer): Boolean = + Try(new ByteBufferParser(buf).parse).isSuccess +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/ChannelSpec.scala b/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/ChannelSpec.scala new file mode 100644 index 00000000..6d5d33a9 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/ChannelSpec.scala @@ -0,0 +1,25 @@ +package jawn +package parser + +import org.scalatest._ + +import java.nio.channels.ByteChannel +import scala.util.Success + +class ChannelSpec extends PropSpec with Matchers { + + property("large strings in files are ok") { + val M = 1000000 + val q = "\"" + val big = q + ("x" * (40 * M)) + q + val bigEscaped = q + ("\\\\" * (20 * M)) + q + + TestUtil.withTemp(big) { t => + Parser.parseFromFile(t)(NullFacade).isSuccess shouldBe true + } + + TestUtil.withTemp(bigEscaped) { t => + Parser.parseFromFile(t)(NullFacade).isSuccess shouldBe true + } + } +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/CharBuilderSpec.scala b/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/CharBuilderSpec.scala new file mode 100644 index 00000000..b25e67fe --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/CharBuilderSpec.scala @@ -0,0 +1,23 @@ +package jawn + +import org.scalatest._ +import org.scalatest.prop._ + +class CharBuilderSpec extends PropSpec with Matchers with PropertyChecks { + + property("append") { + forAll { xs: List[Char] => + val builder = new CharBuilder + xs.foreach(builder.append) + builder.makeString shouldBe xs.mkString + } + } + + property("extend") { + forAll { xs: List[String] => + val builder = new CharBuilder + xs.foreach(builder.extend) + builder.makeString shouldBe xs.mkString + } + } +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/JNumIndexCheck.scala b/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/JNumIndexCheck.scala new file mode 100644 index 00000000..b0b6568d --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/JNumIndexCheck.scala @@ -0,0 +1,81 @@ +package jawn +package parser + +import java.nio.ByteBuffer +import org.scalatest.{Matchers, PropSpec} +import org.scalatest.prop.PropertyChecks +import scala.util.Success + +class JNumIndexCheck extends PropSpec with Matchers with PropertyChecks { + object JNumIndexCheckFacade extends Facade[Boolean] { + class JNumIndexCheckContext(val isObj: Boolean) extends FContext[Boolean] { + var failed = false + def add(s: CharSequence): Unit = () + def add(v: Boolean): Unit = { + if (!v) failed = true + } + def finish: Boolean = !failed + } + + val singleContext: FContext[Boolean] = new JNumIndexCheckContext(false) + val arrayContext: FContext[Boolean] = new JNumIndexCheckContext(false) + val objectContext: FContext[Boolean] = new JNumIndexCheckContext(true) + + def jnull(): Boolean = true + def jfalse(): Boolean = true + def jtrue(): Boolean = true + def jnum(s: CharSequence, decIndex: Int, expIndex: Int): Boolean = { + val input = s.toString + val inputDecIndex = input.indexOf('.') + val inputExpIndex = if (input.indexOf('e') == -1) input.indexOf("E") else input.indexOf('e') + + decIndex == inputDecIndex && expIndex == inputExpIndex + } + def jstring(s: CharSequence): Boolean = true + } + + property("jnum provides the correct indices with parseFromString") { + forAll { (value: BigDecimal) => + val json = s"""{ "num": ${value.toString} }""" + Parser.parseFromString(json)(JNumIndexCheckFacade) shouldBe Success(true) + } + } + + property("jnum provides the correct indices with parseFromByteBuffer") { + forAll { (value: BigDecimal) => + val json = s"""{ "num": ${value.toString} }""" + val bb = ByteBuffer.wrap(json.getBytes("UTF-8")) + Parser.parseFromByteBuffer(bb)(JNumIndexCheckFacade) shouldBe Success(true) + } + } + + property("jnum provides the correct indices with parseFromFile") { + forAll { (value: BigDecimal) => + val json = s"""{ "num": ${value.toString} }""" + TestUtil.withTemp(json) { t => + Parser.parseFromFile(t)(JNumIndexCheckFacade) shouldBe Success(true) + } + } + } + + property("jnum provides the correct indices at the top level with parseFromString") { + forAll { (value: BigDecimal) => + Parser.parseFromString(value.toString)(JNumIndexCheckFacade) shouldBe Success(true) + } + } + + property("jnum provides the correct indices at the top level with parseFromByteBuffer") { + forAll { (value: BigDecimal) => + val bb = ByteBuffer.wrap(value.toString.getBytes("UTF-8")) + Parser.parseFromByteBuffer(bb)(JNumIndexCheckFacade) shouldBe Success(true) + } + } + + property("jnum provides the correct indices at the top level with parseFromFile") { + forAll { (value: BigDecimal) => + TestUtil.withTemp(value.toString) { t => + Parser.parseFromFile(t)(JNumIndexCheckFacade) shouldBe Success(true) + } + } + } +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/SyntaxCheck.scala b/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/SyntaxCheck.scala new file mode 100644 index 00000000..fd00c260 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/SyntaxCheck.scala @@ -0,0 +1,131 @@ +package jawn +package parser + +import org.scalatest._ +import prop._ +import org.scalacheck.Arbitrary._ +import org.scalacheck._ +import Gen._ +import Arbitrary.arbitrary + +import scala.util.{Try, Success, Failure} + +class SyntaxCheck extends PropSpec with Matchers with PropertyChecks { + + sealed trait J { + def build: String = this match { + case JAtom(s) => s + case JArray(js) => js.map(_.build).mkString("[", ",", "]") + case JObject(js) => js.map { case (k, v) => + val kk = "\"" + k + "\"" + val vv = v.build + s"$kk: $vv" + }.mkString("{", ",", "}") + } + } + + case class JAtom(s: String) extends J + case class JArray(js: List[J]) extends J + case class JObject(js: Map[String, J]) extends J + + val jatom: Gen[JAtom] = + Gen.oneOf( + "null", "true", "false", "1234", "-99", "16.0", "2e9", + "-4.44E-10", "11e+14", "\"foo\"", "\"\"", "\"bar\"", + "\"qux\"", "\"duh\"", "\"abc\"", "\"xyz\"", "\"zzzzzz\"", + "\"\\u1234\"").map(JAtom(_)) + + def jarray(lvl: Int): Gen[JArray] = + Gen.containerOf[List, J](jvalue(lvl + 1)).map(JArray(_)) + + val keys = Gen.oneOf("foo", "bar", "qux", "abc", "def", "xyz") + def jitem(lvl: Int): Gen[(String, J)] = + for { s <- keys; j <- jvalue(lvl) } yield (s, j) + + def jobject(lvl: Int): Gen[JObject] = + Gen.containerOf[List, (String, J)](jitem(lvl + 1)).map(ts => JObject(ts.toMap)) + + def jvalue(lvl: Int): Gen[J] = + if (lvl < 3) { + Gen.frequency((16, 'ato), (1, 'arr), (2, 'obj)).flatMap { + case 'ato => jatom + case 'arr => jarray(lvl) + case 'obj => jobject(lvl) + } + } else { + jatom + } + + implicit lazy val arbJValue: Arbitrary[J] = + Arbitrary(jvalue(0)) + + import java.nio.ByteBuffer + + def isValidSyntax(s: String): Boolean = { + val cs = java.nio.CharBuffer.wrap(s.toCharArray) + val r0 = Parser.parseFromCharSequence(cs)(NullFacade).isSuccess + val r1 = Parser.parseFromString(s)(NullFacade).isSuccess + val bb = ByteBuffer.wrap(s.getBytes("UTF-8")) + val r2 = Parser.parseFromByteBuffer(bb)(NullFacade).isSuccess + if (r0 == r1) r1 else sys.error(s"CharSequence/String parsing disagree($r0, $r1): $s") + if (r1 == r2) r1 else sys.error(s"String/ByteBuffer parsing disagree($r1, $r2): $s") + + TestUtil.withTemp(s) { t => + Parser.parseFromFile(t)(NullFacade).isSuccess + } + + val async = AsyncParser[Unit](AsyncParser.SingleValue) + val r3 = async.absorb(s)(NullFacade).isRight && async.finish()(NullFacade).isRight + if (r1 == r3) r1 else sys.error(s"Sync/Async parsing disagree($r1, $r3): $s") + } + + property("syntax-checking") { + forAll { (j: J) => isValidSyntax(j.build) shouldBe true } + } + + def qs(s: String): String = "\"" + s + "\"" + + property("unicode is ok") { + isValidSyntax(qs("ö")) shouldBe true + isValidSyntax(qs("ö\\\\")) shouldBe true + isValidSyntax(qs("\\\\ö")) shouldBe true + } + + property("literal TAB is invalid") { isValidSyntax(qs("\t")) shouldBe false } + property("literal NL is invalid") { isValidSyntax(qs("\n")) shouldBe false } + property("literal CR is invalid") { isValidSyntax(qs("\r")) shouldBe false } + property("literal NUL is invalid") { isValidSyntax(qs("\u0000")) shouldBe false } + property("literal BS TAB is invalid") { isValidSyntax(qs("\\\t")) shouldBe false } + property("literal BS NL is invalid") { isValidSyntax(qs("\\\n")) shouldBe false } + property("literal BS CR is invalid") { isValidSyntax(qs("\\\r")) shouldBe false } + property("literal BS NUL is invalid") { isValidSyntax(qs("\\\u0000")) shouldBe false } + property("literal BS ZERO is invalid") { isValidSyntax(qs("\\0")) shouldBe false } + property("literal BS X is invalid") { isValidSyntax(qs("\\x")) shouldBe false } + + property("0 is ok") { isValidSyntax("0") shouldBe true } + property("0e is invalid") { isValidSyntax("0e") shouldBe false } + property("123e is invalid") { isValidSyntax("123e") shouldBe false } + property(".999 is invalid") { isValidSyntax(".999") shouldBe false } + property("0.999 is ok") { isValidSyntax("0.999") shouldBe true } + property("-.999 is invalid") { isValidSyntax("-.999") shouldBe false } + property("-0.999 is ok") { isValidSyntax("-0.999") shouldBe true } + property("+0.999 is invalid") { isValidSyntax("+0.999") shouldBe false } + property("--0.999 is invalid") { isValidSyntax("--0.999") shouldBe false } + property("01 is invalid") { isValidSyntax("01") shouldBe false } + property("1e is invalid") { isValidSyntax("1e") shouldBe false } + property("1e- is invalid") { isValidSyntax("1e+") shouldBe false } + property("1e+ is invalid") { isValidSyntax("1e-") shouldBe false } + property("1. is invalid") { isValidSyntax("1.") shouldBe false } + property("1.e is invalid") { isValidSyntax("1.e") shouldBe false } + property("1.e9 is invalid") { isValidSyntax("1.e9") shouldBe false } + property("1.e- is invalid") { isValidSyntax("1.e+") shouldBe false } + property("1.e+ is invalid") { isValidSyntax("1.e-") shouldBe false } + property("1.1e is invalid") { isValidSyntax("1.1e") shouldBe false } + property("1.1e- is invalid") { isValidSyntax("1.1e-") shouldBe false } + property("1.1e+ is invalid") { isValidSyntax("1.1e+") shouldBe false } + property("1.1e1 is ok") { isValidSyntax("1.1e1") shouldBe true } + property("1.1e-1 is ok") { isValidSyntax("1.1e-1") shouldBe true } + property("1.1e+1 is ok") { isValidSyntax("1.1e+1") shouldBe true } + property("1+ is invalid") { isValidSyntax("1+") shouldBe false } + property("1- is invalid") { isValidSyntax("1-") shouldBe false } +} diff --git a/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/TestUtil.scala b/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/TestUtil.scala new file mode 100644 index 00000000..64b8dd59 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/parser/src/test/scala/jawn/TestUtil.scala @@ -0,0 +1,18 @@ +package jawn +package parser + +import java.io._ + +object TestUtil { + def withTemp[A](s: String)(f: File => A): A = { + val t = File.createTempFile("jawn-syntax", ".json") + val pw = new PrintWriter(t) + pw.println(s) + pw.close() + try { + f(t) + } finally { + t.delete() + } + } +} diff --git a/scalaplugin/src/test/resource/jawn/project/ReleaseHelper.scala b/scalaplugin/src/test/resource/jawn/project/ReleaseHelper.scala new file mode 100644 index 00000000..354d6506 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/project/ReleaseHelper.scala @@ -0,0 +1,34 @@ +import sbt._ +import sbt.Keys._ +import sbt.complete.Parser + +object ReleaseHelper { + + /** Invoke a command and carry out remaining commands until completion. + * + * This is necessary because sbt-release's releaseStepCommand does not + * execute remaining commands, which sbt-doge relies on. + * + * Based on https://github.com/playframework/playframework/blob/master/framework/project/Release.scala + * + * NOTE: This can be removed in favor of https://github.com/sbt/sbt-release/pull/171 if/when merged upstream + */ + def runCommandAndRemaining(command: String): State => State = { originalState => + val originalRemaining = originalState.remainingCommands + + @annotation.tailrec + def runCommand(command: String, state: State): State = { + val newState = Parser.parse(command, state.combinedParser) match { + case Right(cmd) => cmd() + case Left(msg) => throw sys.error(s"Invalid programmatic input:\n$msg") + } + if (newState.remainingCommands.isEmpty) { + newState + } else { + runCommand(newState.remainingCommands.head, newState.copy(remainingCommands = newState.remainingCommands.tail)) + } + } + + runCommand(command, originalState.copy(remainingCommands = Nil)).copy(remainingCommands = originalRemaining) + } +} diff --git a/scalaplugin/src/test/resource/jawn/project/build.properties b/scalaplugin/src/test/resource/jawn/project/build.properties new file mode 100644 index 00000000..64317fda --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/project/build.properties @@ -0,0 +1 @@ +sbt.version=0.13.15 diff --git a/scalaplugin/src/test/resource/jawn/project/plugins.sbt b/scalaplugin/src/test/resource/jawn/project/plugins.sbt new file mode 100644 index 00000000..618876a9 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/project/plugins.sbt @@ -0,0 +1,6 @@ +addSbtPlugin("com.eed3si9n" % "sbt-doge" % "0.1.5") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.2.25") +addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.1.14") +addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.1") +addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.5") +addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1") diff --git a/scalaplugin/src/test/resource/jawn/randjson.py b/scalaplugin/src/test/resource/jawn/randjson.py new file mode 100644 index 00000000..1783eba5 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/randjson.py @@ -0,0 +1,78 @@ +import json +import os +from random import * +import string +import sys + +constants = [True, False, None] + +def mkconstant(): + return choice(constants) + +def mkinteger(): + return randint(-1e3, 1e3) * (10 ** normalvariate(0, 4)) + randint(-1e3, 1e3) + +def mkdouble(): + return random() * (10 ** normalvariate(0, 30)) + +def mknum(): + if randint(0, 1): + return mkdouble() + else: + return mkinteger() + +def mkstring(): + n = int(min(abs(normalvariate(40, 20)), abs(normalvariate(30, 10)))) + return ''.join([choice(string.ascii_letters) for i in range(0, n)]) + +values = [mkconstant, mknum, mknum, mknum, mkstring] + +def mkvalue(): + return choice(values)() + +def mkarray(n, t, threshold): + a = [] + t2 = t + random() + if (t > threshold): + for i in range(0, 2 * n): + a.append(mkvalue()) + else: + #print "mkarray(%s, %s, %s)" % (n, t, threshold) + for i in range(0, n / 5): + a.append(mkcontainer(t2, threshold)) + return a + +def mkobject(n, t, threshold): + d = {} + t2 = t + random() + if (t > threshold): + for i in range(0, n): + k = mkstring() + v = mkvalue() + d[k] = v + else: + #print "mkobject(%s, %s, %s)" % (n, t, threshold) + for i in range(0, n / 10): + k = mkstring() + v = mkcontainer(t2, threshold) + d[k] = v + return d + +containers = [mkarray, mkobject, mkobject] + +def mkcontainer(t, threshold): + n = int(abs(normalvariate(10, 30))) + return choice(containers)(n, t, threshold) + +if __name__ == "__main__": + args = sys.argv[1:] + try: + weight = float(args[0]) + path = args[1] + print "generating random JSON with weight %s into %s" % (weight, path) + f = open(path, 'w') + c = mkcontainer(0.0, weight) + f.write(json.dumps(c)) + f.close() + except: + print "usage: %s WEIGHT (0.0 < w < ~4.0) FILE" % sys.argv[0] diff --git a/scalaplugin/src/test/resource/jawn/randjson2.py b/scalaplugin/src/test/resource/jawn/randjson2.py new file mode 100644 index 00000000..eb6b9a3a --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/randjson2.py @@ -0,0 +1,53 @@ +import json +import os +from random import * +import string +import sys + +constants = [True, False, None] + +def mkconstant(): + return choice(constants) + +def mkinteger(): + return randint(-1e3, 1e3) * (10 ** normalvariate(0, 4)) + randint(-1e3, 1e3) + +def mkdouble(): + return random() * (10 ** normalvariate(0, 30)) + +def mknum(): + if randint(0, 1): + return mkdouble() + else: + return mkinteger() + +def mkstring(): + n = int(min(abs(normalvariate(40, 20)), abs(normalvariate(30, 10)))) + return ''.join([choice(string.ascii_letters) for i in range(0, n)]) + +values = [mkconstant, mknum, mknum, mknum, mkstring] + +def mkvalue(): + return choice(values)() + +if __name__ == "__main__": + args = sys.argv[1:] + try: + num = int(args[0]) + path = args[1] + print "writing json (%d rows) into %s" % (num, path) + f = open(path, 'w') + f.write("[") + for i in range(0, num): + if i > 0: f.write(", ") + c = {"foo": mkstring(), + "bar": mknum(), + "qux": mkvalue(), + "duh": {"a": mknum(), "b": mknum(), "c": mknum()}, + "xyz": {"yy": mkstring(), "zz": mkvalue()}, + "abc": [mkvalue() for i in range(0, 4)]} + f.write(json.dumps(c)) + f.write("]") + f.close() + except Exception, e: + print "usage: %s NUM PATH" % sys.argv[0] diff --git a/scalaplugin/src/test/resource/jawn/support/argonaut/src/main/scala/Parser.scala b/scalaplugin/src/test/resource/jawn/support/argonaut/src/main/scala/Parser.scala new file mode 100644 index 00000000..0c57e4d7 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/support/argonaut/src/main/scala/Parser.scala @@ -0,0 +1,45 @@ +package jawn +package support.argonaut + +import scala.collection.mutable +import argonaut._ + +object Parser extends SupportParser[Json] { + implicit val facade: Facade[Json] = + new Facade[Json] { + def jnull() = Json.jNull + def jfalse() = Json.jFalse + def jtrue() = Json.jTrue + + def jnum(s: CharSequence, decIndex: Int, expIndex: Int) = + Json.jNumber(JsonNumber.unsafeDecimal(s.toString)) + def jstring(s: CharSequence) = Json.jString(s.toString) + + def singleContext() = new FContext[Json] { + var value: Json = null + def add(s: CharSequence) { value = jstring(s) } + def add(v: Json) { value = v } + def finish: Json = value + def isObj: Boolean = false + } + + def arrayContext() = new FContext[Json] { + val vs = mutable.ListBuffer.empty[Json] + def add(s: CharSequence) { vs += jstring(s) } + def add(v: Json) { vs += v } + def finish: Json = Json.jArray(vs.toList) + def isObj: Boolean = false + } + + def objectContext() = new FContext[Json] { + var key: String = null + var vs = JsonObject.empty + def add(s: CharSequence): Unit = + if (key == null) { key = s.toString } else { vs = vs + (key, jstring(s)); key = null } + def add(v: Json): Unit = + { vs = vs + (key, v); key = null } + def finish = Json.jObject(vs) + def isObj = true + } + } +} diff --git a/scalaplugin/src/test/resource/jawn/support/argonaut/src/test/scala/ParserSpec.scala b/scalaplugin/src/test/resource/jawn/support/argonaut/src/test/scala/ParserSpec.scala new file mode 100644 index 00000000..bb6a8566 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/support/argonaut/src/test/scala/ParserSpec.scala @@ -0,0 +1,41 @@ +package jawn +package support.argonaut + +import argonaut._ +import Argonaut._ +import org.scalacheck.Arbitrary +import org.scalacheck.Arbitrary.arbitrary +import org.scalatest.prop.Checkers +import org.scalatest.{Matchers, FlatSpec} +import scala.util.Try + +object ParserSpec { + case class Example(a: Int, b: Long, c: Double) + + val exampleCodecJson: CodecJson[Example] = + casecodec3(Example.apply, Example.unapply)("a", "b", "c") + + implicit val exampleCaseClassArbitrary: Arbitrary[Example] = Arbitrary( + for { + a <- arbitrary[Int] + b <- arbitrary[Long] + c <- arbitrary[Double] + } yield Example(a, b, c) + ) +} + +class ParserSpec extends FlatSpec with Matchers with Checkers { + import ParserSpec._ + import jawn.support.argonaut.Parser.facade + + "The Argonaut support Parser" should "correctly marshal case classes with Long values" in { + check { (e: Example) => + val jsonString: String = exampleCodecJson.encode(e).nospaces + val json: Try[Json] = jawn.Parser.parseFromString(jsonString) + exampleCodecJson.decodeJson(json.get).toOption match { + case None => fail() + case Some(example) => example == e + } + } + } +} diff --git a/scalaplugin/src/test/resource/jawn/support/json4s/src/main/scala/Parser.scala b/scalaplugin/src/test/resource/jawn/support/json4s/src/main/scala/Parser.scala new file mode 100644 index 00000000..e552621c --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/support/json4s/src/main/scala/Parser.scala @@ -0,0 +1,59 @@ +package jawn +package support.json4s + +import scala.collection.mutable +import org.json4s.JsonAST._ + +object Parser extends Parser(false, false) + +class Parser(useBigDecimalForDouble: Boolean, useBigIntForLong: Boolean) extends SupportParser[JValue] { + + implicit val facade: Facade[JValue] = + new Facade[JValue] { + def jnull() = JNull + def jfalse() = JBool(false) + def jtrue() = JBool(true) + + def jnum(s: CharSequence, decIndex: Int, expIndex: Int) = + if (decIndex == -1 && expIndex == -1) { + if (useBigIntForLong) JInt(BigInt(s.toString)) + else JLong(util.parseLongUnsafe(s)) + } else { + if (useBigDecimalForDouble) JDecimal(BigDecimal(s.toString)) + else JDouble(s.toString.toDouble) + } + + def jstring(s: CharSequence) = JString(s.toString) + + def singleContext() = + new FContext[JValue] { + var value: JValue = null + def add(s: CharSequence) { value = jstring(s) } + def add(v: JValue) { value = v } + def finish: JValue = value + def isObj: Boolean = false + } + + def arrayContext() = + new FContext[JValue] { + val vs = mutable.ListBuffer.empty[JValue] + def add(s: CharSequence) { vs += jstring(s) } + def add(v: JValue) { vs += v } + def finish: JValue = JArray(vs.toList) + def isObj: Boolean = false + } + + def objectContext() = + new FContext[JValue] { + var key: String = null + val vs = mutable.ListBuffer.empty[JField] + def add(s: CharSequence): Unit = + if (key == null) key = s.toString + else { vs += JField(key, jstring(s)); key = null } + def add(v: JValue): Unit = + { vs += JField(key, v); key = null } + def finish: JValue = JObject(vs.toList) + def isObj: Boolean = true + } + } +} diff --git a/scalaplugin/src/test/resource/jawn/support/play/src/main/scala/Parser.scala b/scalaplugin/src/test/resource/jawn/support/play/src/main/scala/Parser.scala new file mode 100644 index 00000000..1bca206a --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/support/play/src/main/scala/Parser.scala @@ -0,0 +1,20 @@ +package jawn +package support.play + +import play.api.libs.json._ + +object Parser extends SupportParser[JsValue] { + + implicit val facade: Facade[JsValue] = + new SimpleFacade[JsValue] { + def jnull() = JsNull + def jfalse() = JsBoolean(false) + def jtrue() = JsBoolean(true) + + def jnum(s: CharSequence, decIndex: Int, expIndex: Int) = JsNumber(BigDecimal(s.toString)) + def jstring(s: CharSequence) = JsString(s.toString) + + def jarray(vs: List[JsValue]) = JsArray(vs) + def jobject(vs: Map[String, JsValue]) = JsObject(vs) + } +} diff --git a/scalaplugin/src/test/resource/jawn/support/rojoma-v3/src/main/scala/Parser.scala b/scalaplugin/src/test/resource/jawn/support/rojoma-v3/src/main/scala/Parser.scala new file mode 100644 index 00000000..c031e71f --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/support/rojoma-v3/src/main/scala/Parser.scala @@ -0,0 +1,18 @@ +package jawn +package support.rojoma.v3 + +import scala.collection.mutable +import com.rojoma.json.v3.ast._ + +object Parser extends SupportParser[JValue] { + implicit val facade: Facade[JValue] = + new MutableFacade[JValue] { + def jnull() = JNull + def jfalse() = JBoolean.canonicalFalse + def jtrue() = JBoolean.canonicalTrue + def jnum(s: CharSequence, decIndex: Int, expIndex: Int) = JNumber.unsafeFromString(s.toString) + def jstring(s: CharSequence) = JString(s.toString) + def jarray(vs: mutable.ArrayBuffer[JValue]) = JArray(vs) + def jobject(vs: mutable.Map[String, JValue]) = JObject(vs) + } +} diff --git a/scalaplugin/src/test/resource/jawn/support/rojoma/src/main/scala/Parser.scala b/scalaplugin/src/test/resource/jawn/support/rojoma/src/main/scala/Parser.scala new file mode 100644 index 00000000..c0725ea3 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/support/rojoma/src/main/scala/Parser.scala @@ -0,0 +1,18 @@ +package jawn +package support.rojoma + +import scala.collection.mutable +import com.rojoma.json.ast._ + +object Parser extends SupportParser[JValue] { + implicit val facade: Facade[JValue] = + new MutableFacade[JValue] { + def jnull() = JNull + def jfalse() = JBoolean.canonicalFalse + def jtrue() = JBoolean.canonicalTrue + def jnum(s: CharSequence, decIndex: Int, expIndex: Int) = JNumber(BigDecimal(s.toString)) + def jstring(s: CharSequence) = JString(s.toString) + def jarray(vs: mutable.ArrayBuffer[JValue]) = JArray(vs) + def jobject(vs: mutable.Map[String, JValue]) = JObject(vs) + } +} diff --git a/scalaplugin/src/test/resource/jawn/support/spray/src/main/scala/Parser.scala b/scalaplugin/src/test/resource/jawn/support/spray/src/main/scala/Parser.scala new file mode 100644 index 00000000..2e589666 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/support/spray/src/main/scala/Parser.scala @@ -0,0 +1,17 @@ +package jawn +package support.spray + +import spray.json._ + +object Parser extends SupportParser[JsValue] { + implicit val facade: Facade[JsValue] = + new SimpleFacade[JsValue] { + def jnull() = JsNull + def jfalse() = JsFalse + def jtrue() = JsTrue + def jnum(s: CharSequence, decIndex: Int, expIndex: Int) = JsNumber(s.toString) + def jstring(s: CharSequence) = JsString(s.toString) + def jarray(vs: List[JsValue]) = JsArray(vs: _*) + def jobject(vs: Map[String, JsValue]) = JsObject(vs) + } +} diff --git a/scalaplugin/src/test/resource/jawn/util/src/main/scala/jawn/util/InvalidLong.scala b/scalaplugin/src/test/resource/jawn/util/src/main/scala/jawn/util/InvalidLong.scala new file mode 100644 index 00000000..adffb979 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/util/src/main/scala/jawn/util/InvalidLong.scala @@ -0,0 +1,7 @@ +package jawn.util + +class InvalidLong(s: String) extends NumberFormatException(s"For input string '$s'") + +object InvalidLong { + def apply(s: String): InvalidLong = new InvalidLong(s) +} diff --git a/scalaplugin/src/test/resource/jawn/util/src/main/scala/jawn/util/Slice.scala b/scalaplugin/src/test/resource/jawn/util/src/main/scala/jawn/util/Slice.scala new file mode 100644 index 00000000..93a8159b --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/util/src/main/scala/jawn/util/Slice.scala @@ -0,0 +1,95 @@ +package jawn.util + +/** + * Character sequence representing a lazily-calculated substring. + * + * This class has three constructors: + * + * - Slice(s) wraps a string, ensuring that future operations + * (e.g. subSequence) will construct slices instead of strings. + * + * - Slice(s, start, limit) is the default, and ensures that: + * + * 1. start >= 0 + * 2. limit >= start + * 3. limit <= s.length + * + * - Slice.unsafe(s, start, limit) is for situations where the above + * bounds-checking has already occurred. Only use this if you are + * absolutely sure your arguments satisfy the above invariants. + * + * Slice's subSequence returns another slice. This means that when + * wrapping a very large string, garbage collection on the underlying + * string will not occur until all slices are freed. + * + * Slice's universal equality is only defined with regard to other + * slices. This means comparing a Slice with other CharSequence values + * (including String) will always return false. + * + * Slices are serializable. However! They use the default Java + * serialization layout, which is not that efficient, and could be a + * disaster in cases where a large shared string might be serialized + * many times in different slices. + */ +@SerialVersionUID(1L) +final class Slice private[jawn] (s: String, start: Int, limit: Int) extends CharSequence with Serializable { + + final val length: Int = + limit - start + + def charAt(i: Int): Char = + if (i < 0 || length <= i) throw new StringIndexOutOfBoundsException(s"index out of range: $i") + else s.charAt(start + i) + + def subSequence(i: Int, j: Int): Slice = + Slice(s, start + i, start + j) + + override def toString: String = + s.substring(start, limit) + + override def equals(that: Any): Boolean = + that match { + case t: AnyRef if this eq t => + true + case slice: Slice => + if (length != slice.length) return false + var i: Int = 0 + while (i < length) { + if (charAt(i) != slice.charAt(i)) return false + i += 1 + } + true + case _ => + false + } + + override def hashCode: Int = { + var hash: Int = 0x90decade + var i: Int = start + while (i < limit) { + hash = s.charAt(i) + (hash * 103696301) // prime + i += 1 + } + hash + } +} + +object Slice { + + val Empty: Slice = Slice("", 0, 0) + + def empty: Slice = Empty + + def apply(s: String): Slice = + new Slice(s, 0, s.length) + + def apply(s: String, start: Int, limit: Int): Slice = + if (start < 0 || limit < start || s.length < limit) { + throw new IndexOutOfBoundsException(s"invalid slice: start=$start, limit=$limit, length=${s.length}") + } else { + new Slice(s, start, limit) + } + + def unsafe(s: String, start: Int, limit: Int): Slice = + new Slice(s, start, limit) +} diff --git a/scalaplugin/src/test/resource/jawn/util/src/main/scala/jawn/util/package.scala b/scalaplugin/src/test/resource/jawn/util/src/main/scala/jawn/util/package.scala new file mode 100644 index 00000000..08f7ae3e --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/util/src/main/scala/jawn/util/package.scala @@ -0,0 +1,96 @@ +package jawn + +package object util { + + /** + * Parse the given character sequence as a single Long value (64-bit + * signed integer) in decimal (base-10). + * + * Other than "0", leading zeros are not allowed, nor are leading + * plusses. At most one leading minus is allowed. The value "-0" is + * allowed, and is interpreted as 0. + * + * Stated more precisely, accepted values: + * + * - conform to the pattern: -?(0|([1-9][0-9]*)) + * - are within [-9223372036854775808, 9223372036854775807] + * + * This method will throw an `InvalidLong` exception on invalid + * input. + */ + def parseLong(cs: CharSequence): Long = { + + // we store the inverse of the positive sum, to ensure we don't + // incorrectly overflow on Long.MinValue. for positive numbers + // this inverse sum will be inverted before being returned. + var inverseSum: Long = 0L + var inverseSign: Long = -1L + var i: Int = 0 + + if (cs.charAt(0) == '-') { + inverseSign = 1L + i = 1 + } + + val len = cs.length + val size = len - i + if (i >= len) throw InvalidLong(cs.toString) + if (size > 19) throw InvalidLong(cs.toString) + if (cs.charAt(i) == '0' && size > 1) throw InvalidLong(cs.toString) + + while (i < len) { + val digit = cs.charAt(i).toInt - 48 + if (digit < 0 || 9 < digit) throw InvalidLong(cs.toString) + inverseSum = inverseSum * 10L - digit + i += 1 + } + + // detect and throw on overflow + if (size == 19 && (inverseSum >= 0 || (inverseSum == Long.MinValue && inverseSign < 0))) { + throw InvalidLong(cs.toString) + } + + inverseSum * inverseSign + } + + /** + * Parse the given character sequence as a single Long value (64-bit + * signed integer) in decimal (base-10). + * + * For valid inputs, this method produces the same values as + * `parseLong`. However, by avoiding input validation it is up to + * 50% faster. + * + * For inputs which `parseLong` throws an error on, + * `parseLongUnsafe` may (or may not) throw an error, or return a + * bogus value. This method makes no guarantees about how it handles + * invalid input. + * + * This method should only be used on sequences which have already + * been parsed (e.g. by a Jawn parser). When in doubt, use + * `parseLong(cs)`, which is still significantly faster than + * `java.lang.Long.parseLong(cs.toString)`. + */ + def parseLongUnsafe(cs: CharSequence): Long = { + + // we store the inverse of the positive sum, to ensure we don't + // incorrectly overflow on Long.MinValue. for positive numbers + // this inverse sum will be inverted before being returned. + var inverseSum: Long = 0L + var inverseSign: Long = -1L + var i: Int = 0 + + if (cs.charAt(0) == '-') { + inverseSign = 1L + i = 1 + } + + val len = cs.length + while (i < len) { + inverseSum = inverseSum * 10L - (cs.charAt(i).toInt - 48) + i += 1 + } + + inverseSum * inverseSign + } +} diff --git a/scalaplugin/src/test/resource/jawn/util/src/test/scala/jawn/util/ParseLongCheck.scala b/scalaplugin/src/test/resource/jawn/util/src/test/scala/jawn/util/ParseLongCheck.scala new file mode 100644 index 00000000..69c4a0e2 --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/util/src/test/scala/jawn/util/ParseLongCheck.scala @@ -0,0 +1,72 @@ +package jawn +package util + +import org.scalatest._ +import prop._ +import org.scalacheck._ + +import scala.util._ + +class ParseLongCheck extends PropSpec with Matchers with PropertyChecks { + + case class UniformLong(value: Long) + + object UniformLong { + implicit val arbitraryUniformLong: Arbitrary[UniformLong] = + Arbitrary(Gen.choose(Long.MinValue, Long.MaxValue).map(UniformLong(_))) + } + + property("both parsers accept on valid input") { + forAll { (n0: UniformLong, prefix: String, suffix: String) => + val n = n0.value + val payload = n.toString + val s = prefix + payload + suffix + val i = prefix.length + val cs = s.subSequence(i, payload.length + i) + cs.toString shouldBe payload + parseLong(cs) shouldBe n + parseLongUnsafe(cs) shouldBe n + } + + forAll { (s: String) => + Try(parseLong(s)) match { + case Success(n) => parseLongUnsafe(s) shouldBe n + case Failure(_) => succeed + } + } + } + + property("safe parser fails on invalid input") { + forAll { (n: Long, m: Long, suffix: String) => + val s1 = n.toString + suffix + Try(parseLong(s1)) match { + case Success(n) => n shouldBe s1.toLong + case Failure(_) => Try(s1.toLong).isFailure + } + + val s2 = n.toString + (m & 0x7fffffffffffffffL).toString + Try(parseLong(s2)) match { + case Success(n) => n shouldBe s2.toLong + case Failure(_) => Try(s2.toLong).isFailure + } + } + + Try(parseLong("9223372036854775807")) shouldBe Try(Long.MaxValue) + Try(parseLong("-9223372036854775808")) shouldBe Try(Long.MinValue) + Try(parseLong("-0")) shouldBe Try(0L) + + assert(Try(parseLong("")).isFailure) + assert(Try(parseLong("+0")).isFailure) + assert(Try(parseLong("00")).isFailure) + assert(Try(parseLong("01")).isFailure) + assert(Try(parseLong("+1")).isFailure) + assert(Try(parseLong("-")).isFailure) + assert(Try(parseLong("--1")).isFailure) + assert(Try(parseLong("9223372036854775808")).isFailure) + assert(Try(parseLong("-9223372036854775809")).isFailure) + } + + // NOTE: parseLongUnsafe is not guaranteed to crash, or do anything + // predictable, on invalid input, so we don't test this direction. + // Its "unsafe" suffix is there for a reason. +} diff --git a/scalaplugin/src/test/resource/jawn/util/src/test/scala/jawn/util/SliceCheck.scala b/scalaplugin/src/test/resource/jawn/util/src/test/scala/jawn/util/SliceCheck.scala new file mode 100644 index 00000000..b56e105e --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/util/src/test/scala/jawn/util/SliceCheck.scala @@ -0,0 +1,131 @@ +package jawn +package util + +import org.scalatest._ +import prop._ +import org.scalacheck._ + +import Arbitrary.arbitrary + +import scala.util._ + +class SliceCheck extends PropSpec with Matchers with PropertyChecks { + + val genSlice: Gen[Slice] = { + val g = arbitrary[String] + def c(start: Int, end: Int): Gen[Int] = + if (end <= start) Gen.const(start) + else Gen.choose(start, end) + Gen.oneOf( + g.map(Slice(_)), + for { s <- g; n = s.length; i <- c(0, n) } yield Slice(s, i, n), + for { s <- g; n = s.length; j <- c(0, n) } yield Slice(s, 0, j), + for { s <- g; n = s.length; i <- c(0, n); j <- c(i, n) } yield Slice(s, i, j)) + } + + implicit val arbitrarySlice: Arbitrary[Slice] = + Arbitrary(genSlice) + + def tryEqual[A](got0: => A, expected0: => A): Unit = { + val got = Try(got0) + val expected = Try(expected0) + got match { + case Success(_) => got shouldBe expected + case Failure(_) => assert(expected.isFailure) + } + } + + property("Slice(s, i, j) ~ s.substring(i, j)") { + forAll { (s: String, i: Int, j: Int) => + tryEqual( + Slice(s, i, j).toString, + s.substring(i, j)) + } + } + + property("Slice(s, i, j).charAt(k) ~ s.substring(i, j).charAt(k)") { + forAll { (s: String, i: Int, j: Int, k: Int) => + tryEqual( + Slice(s, i, j).charAt(k), + s.substring(i, j).charAt(k)) + } + } + + property("slice.length >= 0") { + forAll { (cs: Slice) => + cs.length should be >= 0 + } + } + + property("slice.charAt(i) ~ slice.toString.charAt(i)") { + forAll { (cs: Slice, i: Int) => + tryEqual( + cs.charAt(i), + cs.toString.charAt(i)) + } + } + + property("Slice(s, i, j).subSequence(k, l) ~ s.substring(i, j).substring(k, l)") { + forAll { (s: String, i: Int, j: Int, k: Int, l: Int) => + tryEqual( + Slice(s, i, j).subSequence(k, l).toString, + s.substring(i, j).substring(k, l)) + } + } + + property("Slice(s) ~ Slice(s, 0, s.length)") { + forAll { (s: String) => + tryEqual( + Slice(s).toString, + Slice(s, 0, s.length).toString) + } + } + + property("Slice(s, i, j) => Slice.unsafe(s, i, j)") { + forAll { (s: String, i: Int, j: Int) => + Try(Slice(s, i, j).toString) match { + case Success(r) => r shouldBe Slice.unsafe(s, i, j).toString + case Failure(_) => succeed + } + } + } + + property("x == x") { + forAll { (x: Slice) => x shouldBe x } + } + + property("(x == y) = (x.toString == y.toString)") { + forAll { (x: Slice, y: Slice) => + (x == y) shouldBe (x.toString == y.toString) + } + } + + property("(x == y) -> (x.## == y.##)") { + forAll { (x: Slice, y: Slice) => + if (x == y) x.## shouldBe y.## + else (x.## == y.##) shouldBe false + } + } + + property("x == Slice(x.toString)") { + forAll { (x: Slice) => + Slice(x.toString) shouldBe x + } + } + + property("slice is serializable") { + import java.io._ + + forAll { (x: Slice) => + val baos = new ByteArrayOutputStream() + val oos = new ObjectOutputStream(baos) + oos.writeObject(x) + oos.close() + val bytes = baos.toByteArray + val bais = new ByteArrayInputStream(bytes) + val ois = new ObjectInputStream(bais) + Try(ois.readObject()) shouldBe Try(x) + ois.close() + } + } +} diff --git a/scalaplugin/src/test/resource/jawn/version.sbt b/scalaplugin/src/test/resource/jawn/version.sbt new file mode 100644 index 00000000..1b9f6b1b --- /dev/null +++ b/scalaplugin/src/test/resource/jawn/version.sbt @@ -0,0 +1 @@ +version in ThisBuild := "0.11.1-SNAPSHOT" diff --git a/scalaplugin/src/test/scala/mill/scalaplugin/JawnTests.scala b/scalaplugin/src/test/scala/mill/scalaplugin/JawnTests.scala new file mode 100644 index 00000000..b48fd853 --- /dev/null +++ b/scalaplugin/src/test/scala/mill/scalaplugin/JawnTests.scala @@ -0,0 +1,91 @@ +package mill.scalaplugin + +import ammonite.ops.ImplicitWd._ +import ammonite.ops._ +import mill.Module +import mill.define.{Cross, Task} +import mill.discover.Discovered +import mill.eval.Result +import utest._ +import mill.util.JsonFormatters._ + +object JawnBuild{ + val Jawn = Cross("2.10.6", "2.11.11", "2.12.3").map(new Jawn(_)) + class Jawn(crossVersion: String) extends Module{ + trait JawnModule extends SbtScalaModule{ outer => + def scalaVersion = crossVersion + override def scalacOptions = Seq( + "-deprecation", + "-optimize", + "-unchecked" + ) + def testProjectDeps: Seq[TestScalaModule] = Nil + object test extends this.Tests{ + override def projectDeps = super.projectDeps ++ testProjectDeps + override def ivyDeps = Seq( + Dep("org.scalatest", "scalatest", "3.0.3"), + Dep("org.scalacheck", "scalacheck", "1.13.5") + ) + def testFramework = "org.scalatest.tools.Framework" + } + } + object Parser extends JawnModule{ + def basePath = JawnTests.srcPath/"parser" + } + object Util extends JawnModule{ + override def projectDeps = Seq(Parser) + override def testProjectDeps = Seq(Parser.test) + def basePath = JawnTests.srcPath/"util" + } + object Ast extends JawnModule{ + override def projectDeps = Seq(Parser, Util) + override def testProjectDeps = Seq(Parser.test, Util.test) + def basePath = JawnTests.srcPath/"ast" + } + class Support(name: String, ivyDeps0: Dep*) extends JawnModule{ + override def projectDeps = Seq[ScalaModule](Parser) + def basePath = JawnTests.srcPath/"support"/"argonaut" + override def ivyDeps = ivyDeps0 + } + object Argonaut extends Support("argonaut", Dep("io.argonaut", "argonaut", "6.2")) + object Json4s extends Support("json4s", Dep("org.json4s", "json4s-ast", "3.5.2")) + + object Play extends Support("play"){ + override def ivyDeps = mill.T{ + scalaBinaryVersion() match{ + case "2.10" => Seq(Dep("com.typesafe.play", "play-json", "2.4.11")) + case "2.11" => Seq(Dep("com.typesafe.play", "play-json", "2.5.15")) + case _ => Seq(Dep("com.typesafe.play", "play-json", "2.6.0")) + } + } + } + + object Rojoma extends Support("rojoma", Dep("com.rojoma", "rojoma-json", "2.4.3")) + object RojomaV3 extends Support("rojoma-v3", Dep("com.rojoma", "rojoma-json-v3", "3.7.2")) + object Spray extends Support("spray", Dep("io.spray", "spray-json", "1.3.3")) + } + +} +object JawnTests extends TestSuite{ + val workspacePath = pwd / 'target / 'workspace / "jawn" + val srcPath = pwd / 'scalaplugin / 'src / 'test / 'resource / "jawn" + val tests = Tests{ + rm(workspacePath) + mkdir(workspacePath/up) + cp(srcPath, workspacePath) + val mapping = Discovered.mapping(JawnBuild) + def eval[T](t: Task[T]) = TestEvaluator.eval(mapping, workspacePath)(t) + + 'test - { + def compileOutput = workspacePath / 'jawn / "2.12.3" / 'Parser / 'compile + def testCompileOutput = workspacePath / 'jawn / "2.12.3" / 'Parser / 'test / 'compile + assert(!exists(compileOutput), !exists(testCompileOutput)) + val Right(_) = eval(JawnBuild.Jawn("2.12.3").Parser.test.test()) + assert( + ls.rec(compileOutput).exists(_.last == "AsyncParser.class"), + ls.rec(testCompileOutput).exists(_.last == "CharBuilderSpec.class") + ) + } + + } +} |