From 31e2cdd834c3be3d59bc23ee2c4eff7874d80159 Mon Sep 17 00:00:00 2001 From: Jakob Odersky Date: Fri, 24 Aug 2018 14:37:50 -0700 Subject: Upgrade sbt-settings to major version 2 (#201) This will affect development workflow: instead of running `sbt release` to tag and publish a new version, the release process is now as follows: 1. Create a git tag on a revision that should be published. The tag must be in the format `v[0-9].*` 2. Push the tag to GitHub `git push --tags` 3. CI will build that tag and publish the resulting binary to our artifactory Since the new sbt settings do not enable advanced or risky language features globally anymore (such as higher kinds, reflective calls and implicit conversions), the other changes in this PR either import language features locally or refactor the code to avoid using them entirely. --- .travis.yml | 5 +- build.sbt | 94 ++++++++++++---------- project/build.properties | 2 +- project/plugins.sbt | 4 +- src/main/scala/xyz/driver/core/Refresh.scala | 20 ++++- src/main/scala/xyz/driver/core/app/init.scala | 1 + src/main/scala/xyz/driver/core/cache.scala | 4 +- src/main/scala/xyz/driver/core/core.scala | 7 +- .../xyz/driver/core/database/Repository.scala | 1 + .../scala/xyz/driver/core/file/S3Storage.scala | 32 ++++---- src/main/scala/xyz/driver/core/messaging/Bus.scala | 1 + src/main/scala/xyz/driver/core/rest/package.scala | 25 +++--- src/main/scala/xyz/driver/core/swagger.scala | 37 +++++---- src/main/scala/xyz/driver/core/time.scala | 1 + 14 files changed, 136 insertions(+), 98 deletions(-) diff --git a/.travis.yml b/.travis.yml index c31881d..7845eae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,9 @@ jdk: - oraclejdk8 scala: - - 2.12.3 + - 2.12.6 script: - echo 'credentials += Credentials("Artifactory Realm", "drivergrp.jfrog.io", "sbt-publisher", sys.env("ARTIFACTORY_PASSWORD"))' > project/credentials.sbt - - "sbt clean +test" + - sbt clean +test + - if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_TAG" =~ ^v[0-9].* ]]; then sbt publish; fi diff --git a/build.sbt b/build.sbt index 7ff5f00..6d6da9e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,45 +1,55 @@ import sbt._ import Keys._ -lazy val akkaHttpV = "10.1.1" - -lazy val core = (project in file(".")) - .driverLibrary("core") - .settings(lintingSettings ++ formatSettings) - .settings(libraryDependencies ++= Seq( - "xyz.driver" %% "tracing" % "0.1.2", - "com.softwaremill.sttp" %% "core" % "1.2.2", - "com.softwaremill.sttp" %% "akka-http-backend" % "1.2.2", - "com.typesafe.akka" %% "akka-actor" % "2.5.13", - "com.typesafe.akka" %% "akka-stream" % "2.5.13", - "com.typesafe.akka" %% "akka-http-core" % akkaHttpV, - "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpV, - "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpV, - "com.pauldijou" %% "jwt-core" % "0.16.0", - "io.kamon" %% "kamon-core" % "1.1.3", - "io.kamon" %% "kamon-statsd" % "1.0.0", - "io.kamon" %% "kamon-system-metrics" % "1.0.0", - "io.kamon" %% "kamon-akka-2.5" % "1.0.0", - "org.scala-lang.modules" %% "scala-async" % "0.9.7", - "org.scalatest" %% "scalatest" % "3.0.5" % "test", - "org.scalacheck" %% "scalacheck" % "1.14.0" % "test", - "org.scalaz" %% "scalaz-core" % "7.2.24", - "com.github.swagger-akka-http" %% "swagger-akka-http" % "0.14.1", - "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0", - "eu.timepit" %% "refined" % "0.9.0", - "com.typesafe.slick" %% "slick" % "3.2.3", - "com.beachape" %% "enumeratum" % "1.5.13", - "org.mockito" % "mockito-core" % "1.9.5" % Test, - "com.amazonaws" % "aws-java-sdk-s3" % "1.11.342", - "com.google.cloud" % "google-cloud-pubsub" % "1.31.0", - "com.google.cloud" % "google-cloud-storage" % "1.31.0", - "com.aliyun.oss" % "aliyun-sdk-oss" % "2.8.2", - "com.aliyun.mns" % "aliyun-sdk-mns" % "1.1.8", - "com.typesafe" % "config" % "1.3.3", - "ch.qos.logback" % "logback-classic" % "1.2.3", - "ch.qos.logback.contrib" % "logback-json-classic" % "0.1.5", - "ch.qos.logback.contrib" % "logback-jackson" % "0.1.5", - "com.googlecode.libphonenumber" % "libphonenumber" % "8.9.7", - "com.neovisionaries" % "nv-i18n" % "1.23", - "javax.xml.bind" % "jaxb-api" % "2.2.8" - )) +lazy val core = project + .in(file(".")) + .enablePlugins(LibraryPlugin) + .settings( + libraryDependencies ++= Seq( + // please keep these sorted alphabetically + "ch.qos.logback" % "logback-classic" % "1.2.3", + "ch.qos.logback.contrib" % "logback-jackson" % "0.1.5", + "ch.qos.logback.contrib" % "logback-json-classic" % "0.1.5", + "com.aliyun.mns" % "aliyun-sdk-mns" % "1.1.8", + "com.aliyun.oss" % "aliyun-sdk-oss" % "2.8.2", + "com.amazonaws" % "aws-java-sdk-s3" % "1.11.342", + "com.beachape" %% "enumeratum" % "1.5.13", + "com.github.swagger-akka-http" %% "swagger-akka-http" % "0.14.1", + "com.google.cloud" % "google-cloud-pubsub" % "1.31.0", + "com.google.cloud" % "google-cloud-storage" % "1.31.0", + "com.googlecode.libphonenumber" % "libphonenumber" % "8.9.7", + "com.neovisionaries" % "nv-i18n" % "1.23", + "com.pauldijou" %% "jwt-core" % "0.16.0", + "com.softwaremill.sttp" %% "akka-http-backend" % "1.2.2", + "com.softwaremill.sttp" %% "core" % "1.2.2", + "com.typesafe" % "config" % "1.3.3", + "com.typesafe.akka" %% "akka-actor" % "2.5.14", + "com.typesafe.akka" %% "akka-http-core" % "10.1.4", + "com.typesafe.akka" %% "akka-http-spray-json" % "10.1.4", + "com.typesafe.akka" %% "akka-http-testkit" % "10.1.4", + "com.typesafe.akka" %% "akka-stream" % "2.5.14", + "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0", + "com.typesafe.slick" %% "slick" % "3.2.3", + "eu.timepit" %% "refined" % "0.9.0", + "io.kamon" %% "kamon-akka-2.5" % "1.0.0", + "io.kamon" %% "kamon-core" % "1.1.3", + "io.kamon" %% "kamon-statsd" % "1.0.0", + "io.kamon" %% "kamon-system-metrics" % "1.0.0", + "javax.xml.bind" % "jaxb-api" % "2.2.8", + "org.mockito" % "mockito-core" % "1.9.5" % "test", + "org.scala-lang.modules" %% "scala-async" % "0.9.7", + "org.scalacheck" %% "scalacheck" % "1.14.0" % "test", + "org.scalatest" %% "scalatest" % "3.0.5" % "test", + "org.scalaz" %% "scalaz-core" % "7.2.24", + "xyz.driver" %% "tracing" % "0.1.2" + ), + scalacOptions in (Compile, doc) ++= Seq( + "-groups", // group similar methods together based on the @group annotation. + "-diagrams", // show classs hierarchy diagrams (requires 'dot' to be available on path) + "-implicits", // add methods "inherited" through implicit conversions + "-sourcepath", + baseDirectory.value.getAbsolutePath, + "-doc-source-url", + s"https://github.com/drivergroup/driver-core/blob/master€{FILE_PATH}.scala" + ) + ) diff --git a/project/build.properties b/project/build.properties index d6e3507..5620cc5 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.1.6 +sbt.version=1.2.1 diff --git a/project/plugins.sbt b/project/plugins.sbt index 6ff98d8..17a573c 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1 @@ -resolvers += "releases" at "https://drivergrp.jfrog.io/drivergrp/releases" - -addSbtPlugin("xyz.driver" % "sbt-settings" % "1.0.15") +addSbtPlugin("xyz.driver" % "sbt-settings" % "2.0.7") diff --git a/src/main/scala/xyz/driver/core/Refresh.scala b/src/main/scala/xyz/driver/core/Refresh.scala index ca28da7..e66b22f 100644 --- a/src/main/scala/xyz/driver/core/Refresh.scala +++ b/src/main/scala/xyz/driver/core/Refresh.scala @@ -8,8 +8,8 @@ import scala.concurrent.duration.Duration /** A single-value asynchronous cache with TTL. * - * Slightly adapted from Twitter's "util" library - * https://github.com/twitter/util/blob/ae0ab09134414438af9dfaa88a4613cecbff4741/util-cache/src/main/scala/com/twitter/cache/Refresh.scala + * Slightly adapted from + * [[https://github.com/twitter/util/blob/ae0ab09134414438af9dfaa88a4613cecbff4741/util-cache/src/main/scala/com/twitter/cache/Refresh.scala Twitter's "util" library]] * * Released under the Apache License 2.0. */ @@ -17,15 +17,27 @@ object Refresh { /** Creates a function that will provide a cached value for a given time-to-live (TTL). * + * It avoids the "thundering herd" problem if multiple requests arrive + * simultanously and the cached value has expired or is unset. + * + * Usage example: * {{{ * def freshToken(): Future[String] = // expensive network call to get an access token - * val getToken: Future[String] = Refresh.every(1.hour)(freshToken()) + * val getToken: () => Future[String] = Refresh.every(1.hour)(freshToken()) * * getToken() // new token is issued * getToken() // subsequent calls use the cached token * // wait 1 hour * getToken() // new token is issued - * }}} */ + * }}} + * + * @param ttl Time-To-Live duration to cache a computed value. + * @param compute Call-by-name operation that eventually computes a value to + * be cached. Note that if the computation (i.e. the future) fails, the value + * is not cached. + * @param ec The execution context in which valeu computations will be run. + * @return A zero-arg function that returns the cached value. + */ def every[A](ttl: Duration)(compute: => Future[A])(implicit ec: ExecutionContext): () => Future[A] = { val ref = new AtomicReference[(Future[A], Instant)]( (Future.failed(new NoSuchElementException("Cached value was never computed")), Instant.MIN) diff --git a/src/main/scala/xyz/driver/core/app/init.scala b/src/main/scala/xyz/driver/core/app/init.scala index f1e80b9..b638fd3 100644 --- a/src/main/scala/xyz/driver/core/app/init.scala +++ b/src/main/scala/xyz/driver/core/app/init.scala @@ -15,6 +15,7 @@ import xyz.driver.tracing.{GoogleTracer, NoTracer, Tracer} import scala.concurrent.ExecutionContext import scala.util.Try +import scala.language.reflectiveCalls object init { diff --git a/src/main/scala/xyz/driver/core/cache.scala b/src/main/scala/xyz/driver/core/cache.scala index 3500a2a..9897c16 100644 --- a/src/main/scala/xyz/driver/core/cache.scala +++ b/src/main/scala/xyz/driver/core/cache.scala @@ -89,8 +89,8 @@ object cache { object AsyncCache { val DEFAULT_CAPACITY: Long = 10000L - val DEFAULT_READ_EXPIRATION: Duration = 10 minutes - val DEFAULT_WRITE_EXPIRATION: Duration = 1 hour + val DEFAULT_READ_EXPIRATION: Duration = 10.minutes + val DEFAULT_WRITE_EXPIRATION: Duration = 1.hour def apply[K <: AnyRef, V <: AnyRef]( name: String, diff --git a/src/main/scala/xyz/driver/core/core.scala b/src/main/scala/xyz/driver/core/core.scala index 72237b9..a654e85 100644 --- a/src/main/scala/xyz/driver/core/core.scala +++ b/src/main/scala/xyz/driver/core/core.scala @@ -7,9 +7,12 @@ import xyz.driver.core.rest.errors.ExternalServiceException import scala.concurrent.{ExecutionContext, Future} -package object core { +// TODO: this package seems too complex, look at all the features we need! +import scala.language.reflectiveCalls +import scala.language.higherKinds +import scala.language.implicitConversions - import scala.language.reflectiveCalls +package object core { def make[T](v: => T)(f: T => Unit): T = { val value = v diff --git a/src/main/scala/xyz/driver/core/database/Repository.scala b/src/main/scala/xyz/driver/core/database/Repository.scala index 31c79ad..5d7f787 100644 --- a/src/main/scala/xyz/driver/core/database/Repository.scala +++ b/src/main/scala/xyz/driver/core/database/Repository.scala @@ -6,6 +6,7 @@ import slick.lifted.{AbstractTable, CanBeQueryCondition, RunnableCompiled} import slick.{lifted => sl} import scala.concurrent.{ExecutionContext, Future} +import scala.language.higherKinds trait Repository { type T[D] diff --git a/src/main/scala/xyz/driver/core/file/S3Storage.scala b/src/main/scala/xyz/driver/core/file/S3Storage.scala index 5158d4d..a869919 100644 --- a/src/main/scala/xyz/driver/core/file/S3Storage.scala +++ b/src/main/scala/xyz/driver/core/file/S3Storage.scala @@ -64,20 +64,24 @@ class S3Storage(s3: AmazonS3, bucket: Name[Bucket], executionContext: ExecutionC def isInSubFolder(path: Path)(fileLink: FileLink) = fileLink.location.toString.replace(path.toString + "/", "").contains("/") - Iterator.continually(s3.listObjectsV2(req)).takeWhile { result => - req.setContinuationToken(result.getNextContinuationToken) - result.isTruncated - } flatMap { result => - result.getObjectSummaries.asScala.toList.map { summary => - FileLink( - Name[File](summary.getKey), - Paths.get(path.toString + "/" + summary.getKey), - Revision[File](summary.getETag), - Time(summary.getLastModified.getTime), - summary.getSize - ) - } filterNot isInSubFolder(path) - } toList + Iterator + .continually(s3.listObjectsV2(req)) + .takeWhile { result => + req.setContinuationToken(result.getNextContinuationToken) + result.isTruncated + } + .flatMap { result => + result.getObjectSummaries.asScala.toList.map { summary => + FileLink( + Name[File](summary.getKey), + Paths.get(path.toString + "/" + summary.getKey), + Revision[File](summary.getETag), + Time(summary.getLastModified.getTime), + summary.getSize + ) + } filterNot isInSubFolder(path) + } + .toList }) override def exists(path: Path): Future[Boolean] = Future { diff --git a/src/main/scala/xyz/driver/core/messaging/Bus.scala b/src/main/scala/xyz/driver/core/messaging/Bus.scala index e2ee76a..599af92 100644 --- a/src/main/scala/xyz/driver/core/messaging/Bus.scala +++ b/src/main/scala/xyz/driver/core/messaging/Bus.scala @@ -2,6 +2,7 @@ package xyz.driver.core package messaging import scala.concurrent._ +import scala.language.higherKinds /** Base trait for representing message buses. * diff --git a/src/main/scala/xyz/driver/core/rest/package.scala b/src/main/scala/xyz/driver/core/rest/package.scala index 7d67138..c778b62 100644 --- a/src/main/scala/xyz/driver/core/rest/package.scala +++ b/src/main/scala/xyz/driver/core/rest/package.scala @@ -191,18 +191,21 @@ object `package` { request.headers.find(_.name == ContextHeaders.StacktraceHeader).fold("")(_.value()).split("->") def extractContextHeaders(request: HttpRequest): Map[String, String] = { - request.headers.filter { h => - h.name === ContextHeaders.AuthenticationTokenHeader || h.name === ContextHeaders.TrackingIdHeader || - h.name === ContextHeaders.PermissionsTokenHeader || h.name === ContextHeaders.StacktraceHeader || - h.name === ContextHeaders.TraceHeaderName || h.name === ContextHeaders.SpanHeaderName || - h.name === ContextHeaders.OriginatingIpHeader || h.name === ContextHeaders.ClientFingerprintHeader - } map { header => - if (header.name === ContextHeaders.AuthenticationTokenHeader) { - header.name -> header.value.stripPrefix(ContextHeaders.AuthenticationHeaderPrefix).trim - } else { - header.name -> header.value + request.headers + .filter { h => + h.name === ContextHeaders.AuthenticationTokenHeader || h.name === ContextHeaders.TrackingIdHeader || + h.name === ContextHeaders.PermissionsTokenHeader || h.name === ContextHeaders.StacktraceHeader || + h.name === ContextHeaders.TraceHeaderName || h.name === ContextHeaders.SpanHeaderName || + h.name === ContextHeaders.OriginatingIpHeader || h.name === ContextHeaders.ClientFingerprintHeader + } + .map { header => + if (header.name === ContextHeaders.AuthenticationTokenHeader) { + header.name -> header.value.stripPrefix(ContextHeaders.AuthenticationHeaderPrefix).trim + } else { + header.name -> header.value + } } - } toMap + .toMap } private[rest] def escapeScriptTags(byteString: ByteString): ByteString = { diff --git a/src/main/scala/xyz/driver/core/swagger.scala b/src/main/scala/xyz/driver/core/swagger.scala index 6567290..0c1e15d 100644 --- a/src/main/scala/xyz/driver/core/swagger.scala +++ b/src/main/scala/xyz/driver/core/swagger.scala @@ -69,24 +69,27 @@ object swagger { chain: util.Iterator[ModelConverter]): Property = { val javaType = Json.mapper().constructType(`type`) - Option(javaType.getRawClass) flatMap { cls => - customProperties.get(cls) - } orElse { - `type` match { - case rt: ReferenceType if isOption(javaType.getRawClass) && chain.hasNext => - val nextType = rt.getContentType - val nextResolved = Option(resolveProperty(nextType, context, annotations, chain)).getOrElse( - chain.next().resolveProperty(nextType, context, annotations, chain)) - nextResolved.setRequired(false) - Option(nextResolved) - case t if chain.hasNext => - val nextResolved = chain.next().resolveProperty(t, context, annotations, chain) - nextResolved.setRequired(true) - Option(nextResolved) - case _ => - Option.empty[Property] + Option(javaType.getRawClass) + .flatMap { cls => + customProperties.get(cls) } - } orNull + .orElse { + `type` match { + case rt: ReferenceType if isOption(javaType.getRawClass) && chain.hasNext => + val nextType = rt.getContentType + val nextResolved = Option(resolveProperty(nextType, context, annotations, chain)).getOrElse( + chain.next().resolveProperty(nextType, context, annotations, chain)) + nextResolved.setRequired(false) + Option(nextResolved) + case t if chain.hasNext => + val nextResolved = chain.next().resolveProperty(t, context, annotations, chain) + nextResolved.setRequired(true) + Option(nextResolved) + case _ => + Option.empty[Property] + } + } + .orNull } @SuppressWarnings(Array("org.wartremover.warts.Null")) diff --git a/src/main/scala/xyz/driver/core/time.scala b/src/main/scala/xyz/driver/core/time.scala index c7a32ad..1622068 100644 --- a/src/main/scala/xyz/driver/core/time.scala +++ b/src/main/scala/xyz/driver/core/time.scala @@ -8,6 +8,7 @@ import java.util.concurrent.TimeUnit import xyz.driver.core.date.Month import scala.concurrent.duration._ +import scala.language.implicitConversions import scala.util.Try object time { -- cgit v1.2.3