From c1836497af19e809ffb60017b28416a68467fa10 Mon Sep 17 00:00:00 2001 From: Felix Mulder Date: Thu, 9 Feb 2017 22:01:57 +0100 Subject: Implement relevant functionality for bot, pagination! --- bot/src/dotty/tools/bot/BotServer.scala | 6 +- bot/src/dotty/tools/bot/PullRequestService.scala | 155 ++++++++++++++++------- bot/src/dotty/tools/bot/model/Github.scala | 28 ++++ bot/test/PRServiceTests.scala | 58 +++++++++ 4 files changed, 200 insertions(+), 47 deletions(-) (limited to 'bot') diff --git a/bot/src/dotty/tools/bot/BotServer.scala b/bot/src/dotty/tools/bot/BotServer.scala index 5ce8a83e7..5f38be15b 100644 --- a/bot/src/dotty/tools/bot/BotServer.scala +++ b/bot/src/dotty/tools/bot/BotServer.scala @@ -7,12 +7,16 @@ import scalaz.concurrent.Task object Main extends ServerApp with PullRequestService { + val user = sys.env("USER") + val token = sys.env("TOKEN") + /** Services mounted to the server */ final val services = prService - override def server(args: List[String]): Task[Server] = + override def server(args: List[String]): Task[Server] = { BlazeBuilder .bindHttp(8080, "localhost") .mountService(services, "/api") .start + } } diff --git a/bot/src/dotty/tools/bot/PullRequestService.scala b/bot/src/dotty/tools/bot/PullRequestService.scala index 8b568b134..1d215cb4f 100644 --- a/bot/src/dotty/tools/bot/PullRequestService.scala +++ b/bot/src/dotty/tools/bot/PullRequestService.scala @@ -1,102 +1,167 @@ package dotty.tools.bot -import org.http4s._ +import org.http4s.{ Status => _, _ } import org.http4s.client.blaze._ import org.http4s.client.Client +import org.http4s.headers.Authorization import scalaz.concurrent.Task +import scala.util.control.NonFatal import io.circe._ import io.circe.generic.auto._ import io.circe.syntax._ import org.http4s.circe._ import org.http4s.dsl._ +import org.http4s.util._ -import github4s.Github -import github4s.jvm.Implicits._ -import github4s.free.domain.{ Commit, Issue } +import model.Github._ trait PullRequestService { + /** Username for authorized admin */ + def user: String + + /** OAuth token needed for user to create statuses */ + def token: String + + /** Pull Request HTTP service */ val prService = HttpService { case request @ POST -> Root => request.as(jsonOf[Issue]).flatMap(checkPullRequest) } - private case class CLASignature( + private[this] lazy val authHeader = { + val creds = BasicCredentials(user, token) + new Authorization(creds) + } + + private final case class CLASignature( user: String, signed: Boolean, version: String, currentVersion: String ) - private case class Status( - state: String, - target_url: String, - description: String, - context: String = "continuous-integration/CLA" - ) - def claUrl(userName: String): String = s"https://www.lightbend.com/contribute/cla/scala/check/$userName" def commitsUrl(prNumber: Int): String = - s"https://api.github.com/repos/lampepfl/dotty/pulls/$prNumber/commits" + s"https://api.github.com/repos/lampepfl/dotty/pulls/$prNumber/commits?per_page=100" + + def statusUrl(sha: String): String = + s"https://api.github.com/repos/lampepfl/dotty/statuses/$sha" def toUri(url: String): Task[Uri] = Uri.fromString(url).fold(Task.fail, Task.now) def getRequest(endpoint: Uri): Task[Request] = Task.now { - Request(uri = endpoint, method = Method.GET) + Request(uri = endpoint, method = Method.GET).putHeaders(authHeader) } def postRequest(endpoint: Uri): Task[Request] = Task.now { - Request(uri = endpoint, method = Method.POST) + Request(uri = endpoint, method = Method.POST).putHeaders(authHeader) } def shutdownClient(client: Client): Task[Unit] = Task.now { client.shutdownNow() } - def users(xs: List[Commit]): Task[Set[String]] = Task.now { - xs.map(_.login).flatten.toSet - } - sealed trait CommitStatus { def commit: Commit def isValid: Boolean } - final case class Valid(commit: Commit) extends CommitStatus { def isValid = true } - final case class Invalid(commit: Commit) extends CommitStatus { def isValid = false } + final case class Valid(user: String, commit: Commit) extends CommitStatus { def isValid = true } + final case class Invalid(user: String, commit: Commit) extends CommitStatus { def isValid = false } + final case class CLAServiceDown(user: String, commit: Commit) extends CommitStatus { def isValid = false } + final case class MissingUser(commit: Commit) extends CommitStatus { def isValid = false } /** Partitions invalid and valid commits */ def checkCLA(xs: List[Commit], httpClient: Client): Task[List[CommitStatus]] = { - def checkUser(commit: Commit): Task[CommitStatus] = for { - endpoint <- toUri(claUrl(commit.login.get)) - claReq <- getRequest(endpoint) - claRes <- httpClient.expect(claReq)(jsonOf[CLASignature]) - res = if (claRes.signed) Valid(commit) else Invalid(commit) - } yield res - - Task.gatherUnordered(xs.filter(_.login.isDefined).map(checkUser)) + def checkUser(user: String, commit: Commit): Task[CommitStatus] = { + val claStatus = for { + endpoint <- toUri(claUrl(user)) + claReq <- getRequest(endpoint) + claRes <- httpClient.expect(claReq)(jsonOf[CLASignature]) + res = if (claRes.signed) Valid(user, commit) else Invalid(user, commit) + } yield res + + claStatus.handleWith { + case NonFatal(e) => + println(e) + Task.now(CLAServiceDown(user, commit)) + } + } + + def checkCommit(commit: Commit, author: Author): Task[CommitStatus] = + author.login.map(checkUser(_, commit)).getOrElse(Task.now(MissingUser(commit))) + + Task.gatherUnordered { + xs.flatMap { + case c @ Commit(_, author, commiter, _) => + if (author == commiter) List(checkCommit(c, author)) + else List( + checkCommit(c, author), + checkCommit(c, commiter) + ) + } + } } - def sendStatuses(xs: List[CommitStatus], httpClient: Client): Task[Unit] = { - def setStatus(cm: CommitStatus): Task[Unit] = for { - endpoint <- toUri(cm.commit.url.replaceAll("git\\/commits", "statuses")) - - target = claUrl(cm.commit.login.getOrElse("")) - state = if (cm.isValid) "success" else "failure" - desc = + def sendStatuses(xs: List[CommitStatus], httpClient: Client): Task[List[StatusResponse]] = { + def setStatus(cm: CommitStatus): Task[StatusResponse] = { + val desc = if (cm.isValid) "User signed CLA" else "User needs to sign cla: https://www.lightbend.com/contribute/cla/scala" - statusReq <- postRequest(endpoint).map(_.withBody(Status(state, target, desc).asJson)) - statusRes <- httpClient.expect(statusReq)(jsonOf[String]) - print <- Task.now(println(statusRes)) - } yield print + val stat = cm match { + case Valid(user, commit) => + Status("success", claUrl(user), desc) + case Invalid(user, commit) => + Status("failure", claUrl(user), desc) + case MissingUser(commit) => + Status("failure", "", "Missing valid github user for this PR") + case CLAServiceDown(user, commit) => + Status("pending", claUrl(user), "CLA Service is down") + } + + for { + endpoint <- toUri(statusUrl(cm.commit.sha)) + req <- postRequest(endpoint).map(_.withBody(stat.asJson)) + res <- httpClient.expect(req)(jsonOf[StatusResponse]) + } yield res + } + + Task.gatherUnordered(xs.map(setStatus)) + } + + private[this] val ExtractLink = """<([^>]+)>; rel="([^"]+)"""".r + def findNext(header: Option[Header]): Option[String] = header.flatMap { header => + val value = header.value + + value + .split(',') + .collect { + case ExtractLink(url, kind) if kind == "next" => + url + } + .headOption + } + + def getCommits(issueNbr: Int, httpClient: Client): Task[List[Commit]] = { + def makeRequest(url: String): Task[List[Commit]] = + for { + endpoint <- toUri(url) + req <- getRequest(endpoint) + res <- httpClient.fetch(req){ res => + val link = CaseInsensitiveString("Link") + val next = findNext(res.headers.get(link)).map(makeRequest).getOrElse(Task.now(Nil)) + + res.as[List[Commit]](jsonOf[List[Commit]]).flatMap(commits => next.map(commits ++ _)) + } + } yield res - Task.gatherUnordered(xs.map(setStatus)).map(_ => ()) + makeRequest(commitsUrl(issueNbr)) } def checkPullRequest(issue: Issue): Task[Response] = { @@ -104,12 +169,10 @@ trait PullRequestService { for { // First get all the commits from the PR - endpoint <- toUri(commitsUrl(issue.number)) - commitsReq <- getRequest(endpoint) - commitsRes <- httpClient.expect(commitsReq)(jsonOf[List[Commit]]) + commits <- getCommits(issue.number, httpClient) - // Then get check the CLA of each commit - statuses <- checkCLA(commitsRes, httpClient) + // Then check the CLA of each commit for both author and committer + statuses <- checkCLA(commits, httpClient) // Send statuses to Github and exit _ <- sendStatuses(statuses, httpClient) diff --git a/bot/src/dotty/tools/bot/model/Github.scala b/bot/src/dotty/tools/bot/model/Github.scala index c089f0cf2..fafa2b86a 100644 --- a/bot/src/dotty/tools/bot/model/Github.scala +++ b/bot/src/dotty/tools/bot/model/Github.scala @@ -12,4 +12,32 @@ object Github { number: Int, pull_request: Option[PullRequest] ) + + case class CommitInfo( + message: String + ) + + case class Commit( + sha: String, + author: Author, + committer: Author, + commit: CommitInfo + ) + + case class Author( + login: Option[String] + ) + + case class Status( + state: String, + target_url: String, + description: String, + context: String = "CLA" + ) + + case class StatusResponse( + url: String, + id: Long, + state: String + ) } diff --git a/bot/test/PRServiceTests.scala b/bot/test/PRServiceTests.scala index a8fdba6e2..202c721e6 100644 --- a/bot/test/PRServiceTests.scala +++ b/bot/test/PRServiceTests.scala @@ -31,4 +31,62 @@ class PRServiceTests extends PullRequestService { assert(issue.pull_request.isDefined, "missing pull request") } + + @Test def canGetAllCommitsFromPR = { + val httpClient = PooledHttp1Client() + val issueNbr = 1941 // has 2 commits: https://github.com/lampepfl/dotty/pull/1941/commits + + val List(c1, c2) = getCommits(issueNbr, httpClient).run + + assertEquals( + "Represent untyped operators as Ident instead of Name", + c1.commit.message.takeWhile(_ != '\n') + ) + + assertEquals( + "Better positions for infix term operations.", + c2.commit.message.takeWhile(_ != '\n') + ) + } + + @Test def canGetMoreThan100Commits = { + val httpClient = PooledHttp1Client() + val issueNbr = 1840 // has >100 commits: https://github.com/lampepfl/dotty/pull/1840/commits + + val numberOfCommits = getCommits(issueNbr, httpClient).run.length + + assert( + numberOfCommits > 100, + s"PR 1840, should have a number of commits greater than 100, but was: $numberOfCommits" + ) + } + + @Test def canCheckCLA = { + val httpClient = PooledHttp1Client() + val validUserCommit = Commit("sha-here", Author(Some("felixmulder")), Author(Some("felixmulder")), CommitInfo("")) + val statuses: List[CommitStatus] = checkCLA(validUserCommit :: Nil, httpClient).run + + assert(statuses.length == 1, s"wrong number of valid statuses: got ${statuses.length}, expected 1") + httpClient.shutdownNow() + } + + @Test def canSetStatus = { + val httpClient = PooledHttp1Client() + val sha = "fa64b4b613fe5e78a5b4185b4aeda89e2f1446ff" + val status = Invalid("smarter", Commit(sha, Author(Some("smarter")), Author(Some("smarter")), CommitInfo(""))) + + val statuses: List[StatusResponse] = sendStatuses(status :: Nil, httpClient).run + + assert( + statuses.length == 1, + s"assumed one status response would be returned, got: ${statuses.length}" + ) + + assert( + statuses.head.state == "failure", + s"status set had wrong state, expected 'failure', got: ${statuses.head.state}" + ) + + httpClient.shutdownNow() + } } -- cgit v1.2.3