From 68505aad4a22686594965bc910e60c8d2b6a95d4 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 19 May 2019 15:15:24 +0800 Subject: Bump ammonite to 1.6.7 (#610) * bump ammonite to 1.6.7 * upgrade all the things * add scalaj-http shims for bootstrapping * wip * tweak-error-message * tweak coursier * . --- scalalib/api/src/ZincWorkerApi.scala | 2 +- scalalib/src/Dep.scala | 34 ++++-- scalalib/src/GenIdeaImpl.scala | 4 +- scalalib/src/JavaModule.scala | 10 +- scalalib/src/Lib.scala | 2 +- scalalib/src/ScalaModule.scala | 9 +- scalalib/src/Versions.scala | 2 +- scalalib/src/ZincWorkerModule.scala | 4 +- .../dependency/metadata/MavenMetadataLoader.scala | 5 +- scalalib/src/publish/SonatypeHttpApi.scala | 116 +++++++++++---------- scalalib/src/publish/SonatypePublisher.scala | 5 +- scalalib/src/publish/settings.scala | 6 +- scalalib/test/src/GenIdeaTests.scala | 3 +- scalalib/test/src/ResolveDepsTests.scala | 3 +- .../metadata/MetadataLoaderFactoryTests.scala | 12 ++- .../dependency/updates/UpdatesFinderTests.scala | 5 +- 16 files changed, 130 insertions(+), 92 deletions(-) (limited to 'scalalib') diff --git a/scalalib/api/src/ZincWorkerApi.scala b/scalalib/api/src/ZincWorkerApi.scala index 70128e8d..790ea274 100644 --- a/scalalib/api/src/ZincWorkerApi.scala +++ b/scalalib/api/src/ZincWorkerApi.scala @@ -60,7 +60,7 @@ object Util{ classPath .find(p => p.toString.endsWith(mavenStylePath) || p.toString.endsWith(ivyStylePath)) - .getOrElse(throw new Exception(s"Cannot find $mavenStylePath or $ivyStylePath")) + .getOrElse(throw new Exception(s"Cannot find $mavenStylePath or $ivyStylePath in ${classPath.mkString("[", ", ", "]")}")) } private val ReleaseVersion = raw"""(\d+)\.(\d+)\.(\d+)""".r diff --git a/scalalib/src/Dep.scala b/scalalib/src/Dep.scala index 714fa21e..59c3be5e 100644 --- a/scalalib/src/Dep.scala +++ b/scalalib/src/Dep.scala @@ -9,16 +9,28 @@ case class Dep(dep: coursier.Dependency, cross: CrossVersion, force: Boolean) { def artifactName(binaryVersion: String, fullVersion: String, platformSuffix: String) = { val suffix = cross.suffixString(binaryVersion, fullVersion, platformSuffix) - dep.module.name + suffix + dep.module.name.value + suffix } def configure(attributes: coursier.Attributes): Dep = copy(dep = dep.copy(attributes = attributes)) def forceVersion(): Dep = copy(force = true) - def exclude(exclusions: (String, String)*) = copy(dep = dep.copy(exclusions = dep.exclusions ++ exclusions)) + def exclude(exclusions: (String, String)*) = copy( + dep = dep.copy( + exclusions = + dep.exclusions ++ + exclusions.map{case (k, v) => (coursier.Organization(k), coursier.ModuleName(v))} + ) + ) def excludeOrg(organizations: String*): Dep = exclude(organizations.map(_ -> "*"): _*) def excludeName(names: String*): Dep = exclude(names.map("*" -> _): _*) def toDependency(binaryVersion: String, fullVersion: String, platformSuffix: String) = - dep.copy(module = dep.module.copy(name = artifactName(binaryVersion, fullVersion, platformSuffix))) - def withConfiguration(configuration: String): Dep = copy(dep = dep.copy(configuration = configuration)) + dep.copy( + module = dep.module.copy( + name = coursier.ModuleName(artifactName(binaryVersion, fullVersion, platformSuffix)) + ) + ) + def withConfiguration(configuration: String): Dep = copy( + dep = dep.copy(configuration = coursier.core.Configuration(configuration)) + ) /** * If scalaVersion is a Dotty version, replace the cross-version suffix @@ -49,14 +61,14 @@ case class Dep(dep: coursier.Dependency, cross: CrossVersion, force: Boolean) { object Dep { - val DefaultConfiguration = "default(compile)" + val DefaultConfiguration = coursier.core.Configuration("default(compile)") implicit def parse(signature: String): Dep = { val parts = signature.split(';') val module = parts.head val attributes = parts.tail.foldLeft(coursier.Attributes()) { (as, s) => s.split('=') match { - case Array("classifier", v) => as.copy(classifier = v) + case Array("classifier", v) => as.copy(classifier = coursier.Classifier(v)) case Array(k, v) => throw new Exception(s"Unrecognized attribute: [$s]") case _ => throw new Exception(s"Unable to parse attribute specifier: [$s]") } @@ -72,7 +84,15 @@ object Dep { }).configure(attributes = attributes) } def apply(org: String, name: String, version: String, cross: CrossVersion, force: Boolean = false): Dep = { - apply(coursier.Dependency(coursier.Module(org, name), version, DefaultConfiguration), cross, force) + apply( + coursier.Dependency( + coursier.Module(coursier.Organization(org), coursier.ModuleName(name)), + version, + DefaultConfiguration + ), + cross, + force + ) } implicit def rw: RW[Dep] = macroRW } diff --git a/scalalib/src/GenIdeaImpl.scala b/scalalib/src/GenIdeaImpl.scala index 379ce30b..404a0235 100755 --- a/scalalib/src/GenIdeaImpl.scala +++ b/scalalib/src/GenIdeaImpl.scala @@ -1,7 +1,7 @@ package mill.scalalib import ammonite.runtime.SpecialClassLoader -import coursier.{Cache, CoursierPaths, Repository} +import coursier.Repository import mill.define._ import mill.eval.{Evaluator, PathRef, Result} import mill.api.Ctx.{Home, Log} @@ -204,7 +204,7 @@ object GenIdeaImpl { // Tries to group jars with their poms and sources. def toResolvedJar(path : os.Path) : Option[ResolvedLibrary] = { - val inCoursierCache = path.startsWith(os.Path(CoursierPaths.cacheDirectory())) + val inCoursierCache = path.startsWith(os.Path(coursier.paths.CoursierPaths.cacheDirectory())) val isSource = path.last.endsWith("sources.jar") val isPom = path.ext == "pom" if (inCoursierCache && (isSource || isPom)) { diff --git a/scalalib/src/JavaModule.scala b/scalalib/src/JavaModule.scala index e1fd8628..7b373650 100644 --- a/scalalib/src/JavaModule.scala +++ b/scalalib/src/JavaModule.scala @@ -383,8 +383,14 @@ trait JavaModule extends mill.Module with TaskModule { outer => Some(mapDependencies()) ) - println(coursier.util.Print.dependencyTree(flattened, resolution, - printExclusions = false, reverse = inverse)) + println( + coursier.util.Print.dependencyTree( + roots = flattened, + resolution = resolution, + printExclusions = false, + reverse = inverse + ) + ) Result.Success() } diff --git a/scalalib/src/Lib.scala b/scalalib/src/Lib.scala index 2706850e..da133d94 100644 --- a/scalalib/src/Lib.scala +++ b/scalalib/src/Lib.scala @@ -8,7 +8,7 @@ import java.util.zip.ZipInputStream import javax.tools.ToolProvider import ammonite.util.Util -import coursier.{Cache, Dependency, Fetch, Repository, Resolution} +import coursier.{Dependency, Fetch, Repository, Resolution} import mill.scalalib.api.Util.isDotty import mill.Agg import mill.eval.{PathRef, Result} diff --git a/scalalib/src/ScalaModule.scala b/scalalib/src/ScalaModule.scala index 5fad1664..f45a7e98 100644 --- a/scalalib/src/ScalaModule.scala +++ b/scalalib/src/ScalaModule.scala @@ -48,8 +48,13 @@ trait ScalaModule extends JavaModule { outer => Set("dotty-library", "dotty-compiler") else Set("scala-library", "scala-compiler", "scala-reflect") - if (!artifacts(d.module.name)) d - else d.copy(module = d.module.copy(organization = scalaOrganization()), version = scalaVersion()) + if (!artifacts(d.module.name.value)) d + else d.copy( + module = d.module.copy( + organization = coursier.Organization(scalaOrganization()) + ), + version = scalaVersion() + ) } override def resolveCoursierDependency: Task[Dep => coursier.Dependency] = T.task{ diff --git a/scalalib/src/Versions.scala b/scalalib/src/Versions.scala index 231eb86b..973b05ed 100644 --- a/scalalib/src/Versions.scala +++ b/scalalib/src/Versions.scala @@ -2,7 +2,7 @@ package mill.scalalib object Versions { // Keep synchronized with ammonite dependency in core in build.sc - val ammonite = "1.6.0" + val ammonite = "1.6.7" // Keep synchronized with zinc dependency in scalalib.worker in build.sc val zinc = "1.2.5" } diff --git a/scalalib/src/ZincWorkerModule.scala b/scalalib/src/ZincWorkerModule.scala index 50d37611..7d81cb97 100644 --- a/scalalib/src/ZincWorkerModule.scala +++ b/scalalib/src/ZincWorkerModule.scala @@ -1,6 +1,6 @@ package mill.scalalib -import coursier.Cache + import coursier.maven.MavenRepository import mill.Agg import mill.T @@ -17,7 +17,7 @@ object ZincWorkerModule extends mill.define.ExternalModule with ZincWorkerModule } trait ZincWorkerModule extends mill.Module{ def repositories = Seq( - Cache.ivy2Local, + coursier.LocalRepositories.ivy2Local, MavenRepository("https://repo1.maven.org/maven2"), MavenRepository("https://oss.sonatype.org/content/repositories/releases") ) diff --git a/scalalib/src/dependency/metadata/MavenMetadataLoader.scala b/scalalib/src/dependency/metadata/MavenMetadataLoader.scala index 491911bf..e40337fc 100644 --- a/scalalib/src/dependency/metadata/MavenMetadataLoader.scala +++ b/scalalib/src/dependency/metadata/MavenMetadataLoader.scala @@ -1,6 +1,5 @@ package mill.scalalib.dependency.metadata -import coursier.Cache import coursier.maven.MavenRepository import coursier.util.Task import mill.scalalib.dependency.versions.Version @@ -8,14 +7,14 @@ import mill.scalalib.dependency.versions.Version private[dependency] final case class MavenMetadataLoader(mavenRepo: MavenRepository) extends MetadataLoader { - private val fetch = Cache.fetch[Task]() + private val fetch = coursier.cache.FileCache[Task].fetch override def getVersions(module: coursier.Module): List[Version] = { import scala.concurrent.ExecutionContext.Implicits.global // TODO fallback to 'versionsFromListing' if 'versions' doesn't work? (needs to be made public in coursier first) val allVersions = mavenRepo.versions(module, fetch).run.unsafeRun allVersions - .map(_.available.map(Version(_))) + .map(_._1.available.map(Version(_))) .getOrElse(List.empty) } } diff --git a/scalalib/src/publish/SonatypeHttpApi.scala b/scalalib/src/publish/SonatypeHttpApi.scala index 12defa93..217d556e 100644 --- a/scalalib/src/publish/SonatypeHttpApi.scala +++ b/scalalib/src/publish/SonatypeHttpApi.scala @@ -5,18 +5,10 @@ import java.util.Base64 import scala.concurrent.duration._ -import scalaj.http.{BaseHttp, HttpOptions, HttpRequest, HttpResponse} - -object PatientHttp - extends BaseHttp( - options = Seq( - HttpOptions.connTimeout(5.seconds.toMillis.toInt), - HttpOptions.readTimeout(1.minute.toMillis.toInt), - HttpOptions.followRedirects(false) - ) - ) + class SonatypeHttpApi(uri: String, credentials: String) { + val http = requests.Session(connectTimeout = 5000, readTimeout = 1000, maxRedirects = 0) private val base64Creds = base64(credentials) @@ -29,12 +21,19 @@ class SonatypeHttpApi(uri: String, credentials: String) { // https://oss.sonatype.org/nexus-staging-plugin/default/docs/path__staging_profiles.html def getStagingProfileUri(groupId: String): String = { val response = withRetry( - PatientHttp(s"$uri/staging/profiles").headers(commonHeaders)) - .throwError + http.get( + s"$uri/staging/profiles", + headers = commonHeaders + ) + ) + + if (!response.is2xx) { + throw new Exception(s"$uri/staging/profiles returned ${response.statusCode}") + } val resourceUri = ujson - .read(response.body)("data") + .read(response.data.text)("data") .arr .find(profile => groupId.split('.').startsWith(profile("name").str.split('.'))) @@ -47,79 +46,84 @@ class SonatypeHttpApi(uri: String, credentials: String) { } def getStagingRepoState(stagingRepoId: String): String = { - val response = PatientHttp(s"${uri}/staging/repository/${stagingRepoId}") - .option(HttpOptions.readTimeout(60000)) - .headers(commonHeaders) - .asString - .throwError - - ujson.read(response.body)("type").str.toString + val response = http.get( + s"${uri}/staging/repository/${stagingRepoId}", + readTimeout = 60000, + headers = commonHeaders + ) + ujson.read(response.data.text)("type").str.toString } // https://oss.sonatype.org/nexus-staging-plugin/default/docs/path__staging_profiles_-profileIdKey-_start.html def createStagingRepo(profileUri: String, groupId: String): String = { - val response = withRetry(PatientHttp(s"${profileUri}/start") - .headers(commonHeaders) - .postData( - s"""{"data": {"description": "fresh staging profile for ${groupId}"}}""")) - .throwError + val response = http.post( + s"${profileUri}/start", + headers = commonHeaders, + data = s"""{"data": {"description": "fresh staging profile for ${groupId}"}}""" + ) - ujson.read(response.body)("data")("stagedRepositoryId").str.toString + if (!response.is2xx) { + throw new Exception(s"$uri/staging/profiles returned ${response.statusCode}") + } + + ujson.read(response.data.text)("data")("stagedRepositoryId").str.toString } // https://oss.sonatype.org/nexus-staging-plugin/default/docs/path__staging_profiles_-profileIdKey-_finish.html def closeStagingRepo(profileUri: String, repositoryId: String): Boolean = { val response = withRetry( - PatientHttp(s"${profileUri}/finish") - .headers(commonHeaders) - .postData( - s"""{"data": {"stagedRepositoryId": "${repositoryId}", "description": "closing staging repository"}}""" - )) + http.post( + s"${profileUri}/finish", + headers = commonHeaders, + data = s"""{"data": {"stagedRepositoryId": "${repositoryId}", "description": "closing staging repository"}}""" + ) + ) - response.code == 201 + response.statusCode == 201 } // https://oss.sonatype.org/nexus-staging-plugin/default/docs/path__staging_profiles_-profileIdKey-_promote.html def promoteStagingRepo(profileUri: String, repositoryId: String): Boolean = { val response = withRetry( - PatientHttp(s"${profileUri}/promote") - .headers(commonHeaders) - .postData( - s"""{"data": {"stagedRepositoryId": "${repositoryId}", "description": "promote staging repository"}}""" - )) + http.post( + s"${profileUri}/promote", + headers = commonHeaders, + data = s"""{"data": {"stagedRepositoryId": "${repositoryId}", "description": "promote staging repository"}}""" + ) + ) - response.code == 201 + response.statusCode == 201 } // https://oss.sonatype.org/nexus-staging-plugin/default/docs/path__staging_profiles_-profileIdKey-_drop.html def dropStagingRepo(profileUri: String, repositoryId: String): Boolean = { val response = withRetry( - PatientHttp(s"${profileUri}/drop") - .headers(commonHeaders) - .postData( - s"""{"data": {"stagedRepositoryId": "${repositoryId}", "description": "drop staging repository"}}""" - )) - - response.code == 201 + http.post( + s"${profileUri}/drop", + headers = commonHeaders, + data = s"""{"data": {"stagedRepositoryId": "${repositoryId}", "description": "drop staging repository"}}""" + ) + ) + response.statusCode == 201 } private val uploadTimeout = 5.minutes.toMillis.toInt - def upload(uri: String, data: Array[Byte]): HttpResponse[String] = { - PatientHttp(uri) - .option(HttpOptions.readTimeout(uploadTimeout)) - .method("PUT") - .headers( + def upload(uri: String, data: Array[Byte]): requests.Response = { + http.put( + uri, + readTimeout = uploadTimeout, + headers = Seq( "Content-Type" -> "application/binary", "Authorization" -> s"Basic ${base64Creds}" - ) - .put(data) - .asString + ), + data = data + ) } - private def withRetry(request: HttpRequest, - retries: Int = 10): HttpResponse[String] = { - val resp = request.asString + private def withRetry(request: => requests.Response, + retries: Int = 10): requests.Response = { + val resp = request if (resp.is5xx && retries > 0) { Thread.sleep(500) withRetry(request, retries - 1) diff --git a/scalalib/src/publish/SonatypePublisher.scala b/scalalib/src/publish/SonatypePublisher.scala index 6dcadd5b..5ca8f5c1 100644 --- a/scalalib/src/publish/SonatypePublisher.scala +++ b/scalalib/src/publish/SonatypePublisher.scala @@ -5,7 +5,6 @@ import java.security.MessageDigest import mill.api.Logger import os.Shellable -import scalaj.http.HttpResponse class SonatypePublisher(uri: String, snapshotUri: String, @@ -105,13 +104,13 @@ class SonatypePublisher(uri: String, } } - private def reportPublishResults(publishResults: Seq[HttpResponse[String]], + private def reportPublishResults(publishResults: Seq[requests.Response], artifacts: Seq[Artifact]) = { if (publishResults.forall(_.is2xx)) { log.info(s"Published ${artifacts.map(_.id).mkString(", ")} to Sonatype") } else { val errors = publishResults.filterNot(_.is2xx).map { response => - s"Code: ${response.code}, message: ${response.body}" + s"Code: ${response.statusCode}, message: ${response.data.text}" } throw new RuntimeException( s"Failed to publish ${artifacts.map(_.id).mkString(", ")} to Sonatype. Errors: \n${errors.mkString("\n")}" diff --git a/scalalib/src/publish/settings.scala b/scalalib/src/publish/settings.scala index bca81cf0..d2801752 100644 --- a/scalalib/src/publish/settings.scala +++ b/scalalib/src/publish/settings.scala @@ -23,13 +23,13 @@ object Artifact { ) Dependency( Artifact( - dep.dep.module.organization, + dep.dep.module.organization.value, name, dep.dep.version ), Scope.Compile, - if (dep.dep.configuration == "") None else Some(dep.dep.configuration), - dep.dep.exclusions.toList + if (dep.dep.configuration == "") None else Some(dep.dep.configuration.value), + dep.dep.exclusions.toList.map{case (a, b) => (a.value, b.value)} ) } } diff --git a/scalalib/test/src/GenIdeaTests.scala b/scalalib/test/src/GenIdeaTests.scala index f8d9a0ed..60c9f9a8 100644 --- a/scalalib/test/src/GenIdeaTests.scala +++ b/scalalib/test/src/GenIdeaTests.scala @@ -1,6 +1,5 @@ package mill.scalalib -import coursier.Cache import mill._ import mill.util.{TestEvaluator, TestUtil} import utest._ @@ -57,6 +56,6 @@ object GenIdeaTests extends TestSuite { private def normaliseLibraryPaths(in: String): String = { - in.replaceAll(Cache.default.toPath.toAbsolutePath.toString, "COURSIER_HOME") + in.replaceAll(coursier.paths.CoursierPaths.cacheDirectory().toString, "COURSIER_HOME") } } diff --git a/scalalib/test/src/ResolveDepsTests.scala b/scalalib/test/src/ResolveDepsTests.scala index ce905907..94b8adb9 100644 --- a/scalalib/test/src/ResolveDepsTests.scala +++ b/scalalib/test/src/ResolveDepsTests.scala @@ -1,6 +1,5 @@ package mill.scalalib -import coursier.Cache import coursier.maven.MavenRepository import mill.api.Result.{Failure, Success} import mill.eval.{PathRef, Result} @@ -8,7 +7,7 @@ import mill.api.Loose.Agg import utest._ object ResolveDepsTests extends TestSuite { - val repos = Seq(Cache.ivy2Local, MavenRepository("https://repo1.maven.org/maven2")) + val repos = Seq(coursier.LocalRepositories.ivy2Local, MavenRepository("https://repo1.maven.org/maven2")) def evalDeps(deps: Agg[Dep]): Result[Agg[PathRef]] = Lib.resolveDependencies( repos, diff --git a/scalalib/test/src/dependency/metadata/MetadataLoaderFactoryTests.scala b/scalalib/test/src/dependency/metadata/MetadataLoaderFactoryTests.scala index 4c2206b8..af2ea617 100644 --- a/scalalib/test/src/dependency/metadata/MetadataLoaderFactoryTests.scala +++ b/scalalib/test/src/dependency/metadata/MetadataLoaderFactoryTests.scala @@ -28,8 +28,8 @@ */ package mill.scalalib.dependency.metadata -import coursier.Fetch.Content -import coursier.core.{Artifact, Module, Project, Repository} +import coursier.Fetch +import coursier.core.{Artifact, Classifier, Dependency, Module, Project, Repository} import coursier.ivy.IvyRepository import coursier.maven.MavenRepository import coursier.util.{EitherT, Monad} @@ -57,8 +57,12 @@ object MetadataLoaderFactoryTests extends TestSuite { } case class CustomRepository() extends Repository { - override def find[F[_]](module: Module, version: String, fetch: Content[F])( - implicit F: Monad[F]): EitherT[F, String, (Artifact.Source, Project)] = + override def find[F[_]](module: Module, version: String, fetch: coursier.Repository.Fetch[F]) + (implicit F: Monad[F]): EitherT[F, String, (Artifact.Source, Project)] = ??? + + override def artifacts(dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[Classifier]]) = ??? } } diff --git a/scalalib/test/src/dependency/updates/UpdatesFinderTests.scala b/scalalib/test/src/dependency/updates/UpdatesFinderTests.scala index 7b6e6e36..3b613bcb 100644 --- a/scalalib/test/src/dependency/updates/UpdatesFinderTests.scala +++ b/scalalib/test/src/dependency/updates/UpdatesFinderTests.scala @@ -37,7 +37,10 @@ object UpdatesFinderTests extends TestSuite { available: Seq[String], allowPreRelease: Boolean) = { val dependency = coursier.Dependency( - coursier.Module("com.example.organization", "example-artifact"), + coursier.Module( + coursier.Organization("com.example.organization"), + coursier.ModuleName("example-artifact") + ), current) val currentVersion = Version(current) val allVersions = available.map(Version(_)).toSet -- cgit v1.2.3