From 26aa8adc30a84d983d020e34b488ac22a31cb544 Mon Sep 17 00:00:00 2001 From: Jakob Odersky Date: Sat, 1 Apr 2017 13:21:15 -0700 Subject: Add yaml parser and docker executor --- .../main/scala/io/crashbox/ci/DockerExecutor.scala | 140 +++++++++++++++++++++ .../src/main/scala/io/crashbox/ci/Parser.scala | 44 +++++++ crashboxd/src/main/scala/io/crashbox/ci/defs.scala | 7 ++ .../io/crashbox/ci/yaml/CompositeReaders.scala | 29 +++++ .../scala/io/crashbox/ci/yaml/DefaultReaders.scala | 6 + .../scala/io/crashbox/ci/yaml/SimpleReaders.scala | 46 +++++++ .../src/main/scala/io/crashbox/ci/yaml/Yaml.scala | 36 ++++++ .../io/crashbox/ci/yaml/YamlFormatException.scala | 4 + .../scala/io/crashbox/ci/yaml/YamlReader.scala | 24 ++++ .../main/scala/io/crashbox/ci/yaml/values.scala | 12 ++ 10 files changed, 348 insertions(+) create mode 100644 crashboxd/src/main/scala/io/crashbox/ci/DockerExecutor.scala create mode 100644 crashboxd/src/main/scala/io/crashbox/ci/Parser.scala create mode 100644 crashboxd/src/main/scala/io/crashbox/ci/defs.scala create mode 100644 crashboxd/src/main/scala/io/crashbox/ci/yaml/CompositeReaders.scala create mode 100644 crashboxd/src/main/scala/io/crashbox/ci/yaml/DefaultReaders.scala create mode 100644 crashboxd/src/main/scala/io/crashbox/ci/yaml/SimpleReaders.scala create mode 100644 crashboxd/src/main/scala/io/crashbox/ci/yaml/Yaml.scala create mode 100644 crashboxd/src/main/scala/io/crashbox/ci/yaml/YamlFormatException.scala create mode 100644 crashboxd/src/main/scala/io/crashbox/ci/yaml/YamlReader.scala create mode 100644 crashboxd/src/main/scala/io/crashbox/ci/yaml/values.scala (limited to 'crashboxd/src/main') diff --git a/crashboxd/src/main/scala/io/crashbox/ci/DockerExecutor.scala b/crashboxd/src/main/scala/io/crashbox/ci/DockerExecutor.scala new file mode 100644 index 0000000..3c24b06 --- /dev/null +++ b/crashboxd/src/main/scala/io/crashbox/ci/DockerExecutor.scala @@ -0,0 +1,140 @@ +package io.crashbox.ci + +import java.io.{File, OutputStream} + +import scala.collection.JavaConverters._ +import scala.concurrent.Future +import scala.concurrent.duration._ + +import akka.actor.ActorSystem +import com.spotify.docker.client.DefaultDockerClient +import com.spotify.docker.client.DockerClient.{ + AttachParameter, + ListContainersParam +} +import com.spotify.docker.client.LogStream +import com.spotify.docker.client.exceptions.ContainerNotFoundException +import com.spotify.docker.client.messages.{ContainerConfig, HostConfig} +import com.spotify.docker.client.messages.HostConfig.Bind + +object DockerExecutor { + + case class ExecutionId(id: String) extends AnyVal + + def containerUser = "crashbox" + def containerWorkDirectory = "/home/crashbox" + def containerKillTimeout = 5.seconds + +} + +class DockerExecutor(uri: String = "unix:///run/docker.sock")( + implicit system: ActorSystem) { + import DockerExecutor._ + import system.log + + val dockerClient = { + val c = DefaultDockerClient.builder().uri(uri).build() + system.registerOnTermination { + c.close() + } + c + } + + /* + def makeImage() = { + val returnedImageId = dockerClient.build( + Paths.get("docker directory"), "test", new ProgressHandler() { + override def progress(message: ProgressMessage) = { + val imageId = message.buildImageId() + message.buildImageId() + if (imageId != null) { + //imageIdFromMessage.set(imageId); + } + } + }) + //dockerClient.build + }*/ + + def start( + image: String, + script: String, + buildDirectory: File, + out: OutputStream + ): Future[ExecutionId] = + Future { + val volume = Bind + .builder() + .from(buildDirectory.getAbsolutePath) + .to(containerWorkDirectory) + .build() + val hostConfig = HostConfig.builder().binds(volume).build() + val containerConfig = ContainerConfig + .builder() + .labels(Map("crashbox" -> "build").asJava) + .hostConfig(hostConfig) + .tty(true) // combine stdout and stderr into stdout + .image(image) + .user(containerUser) + .workingDir(containerWorkDirectory) + .entrypoint("/bin/sh", "-c") + .cmd(script) + .build() + val container = dockerClient.createContainer(containerConfig).id + + log.debug(s"Starting container $container") + dockerClient.startContainer(container) + + log.debug(s"Attaching log stream of container $container") + system.dispatcher execute new Runnable { + override def run() = { + var stream: LogStream = null + try { + stream = dockerClient.attachContainer( + container, + AttachParameter.LOGS, + AttachParameter.STDOUT, + AttachParameter.STREAM + ) + stream.attach(out, null, true) + } finally { + if (stream != null) stream.close() + } + } + } + ExecutionId(container) + }(system.dispatcher) + + def result(id: ExecutionId): Future[Int] = + Future { + log.debug(s"Waiting for container $id to exit") + val res: Int = dockerClient.waitContainer(id.id).statusCode() + stop(id) + res + }(system.dispatcher) + + def stop(id: ExecutionId): Unit = { + try { + log.debug(s"Stopping container $id") + dockerClient.stopContainer(id.id, + containerKillTimeout.toUnit(SECONDS).toInt) + log.debug(s"Removing container $id") + dockerClient.removeContainer(id.id) + } catch { + case _: ContainerNotFoundException => // build already cancelled + } + } + + def reapDeadBuilds(): Unit = { + val stale = dockerClient + .listContainers( + ListContainersParam.withLabel("crashbox"), + ListContainersParam.withStatusExited() + ) + .asScala + stale.foreach { container => + log.warning(s"Removing stale container ${container.id}") + dockerClient.removeContainer(container.id) + } + } + +} diff --git a/crashboxd/src/main/scala/io/crashbox/ci/Parser.scala b/crashboxd/src/main/scala/io/crashbox/ci/Parser.scala new file mode 100644 index 0000000..5a58612 --- /dev/null +++ b/crashboxd/src/main/scala/io/crashbox/ci/Parser.scala @@ -0,0 +1,44 @@ +package io.crashbox.ci + +import io.crashbox.ci.yaml._ +import io.crashbox.ci.yaml.DefaultReaders._ + +object Parser { + sealed trait Result + case class Success(buildDef: BuildDef) extends Result + case class Failure(error: String) extends Result + + implicit object TaskDefReader extends YamlReader[TaskDef] { + def read(value: YamlValue) = { + val items = value.convertTo[Map[String, YamlValue]] + val image = items + .getOrElse("image", + throw new YamlFormatException("no image specified")) + .convertTo[String] + val script = items + .getOrElse("script", + throw new YamlFormatException("no script specified")) + .convertTo[String] + TaskDef(DockerEnvironment(image), script) + } + } + + implicit object BuildDefReader extends YamlReader[BuildDef] { + def read(value: YamlValue) = { + val items = value.convertTo[Map[String, YamlValue]] + val tasks = items + .getOrElse("tasks", + throw new YamlFormatException("no tasks specified")) + .convertTo[Map[String, TaskDef]] + BuildDef(tasks.values.toSeq) + } + } + + def parse(build: String): Result = + try { + Success(Yaml.parse(build).convertTo[BuildDef]) + } catch { + case ex: YamlFormatException => Failure(ex.toString) + } + +} diff --git a/crashboxd/src/main/scala/io/crashbox/ci/defs.scala b/crashboxd/src/main/scala/io/crashbox/ci/defs.scala new file mode 100644 index 0000000..df014a0 --- /dev/null +++ b/crashboxd/src/main/scala/io/crashbox/ci/defs.scala @@ -0,0 +1,7 @@ +package io.crashbox.ci + +sealed trait Environment +case class DockerEnvironment(image: String) extends Environment + +case class TaskDef(environment: Environment, script: String) +case class BuildDef(tasks: Seq[TaskDef]) diff --git a/crashboxd/src/main/scala/io/crashbox/ci/yaml/CompositeReaders.scala b/crashboxd/src/main/scala/io/crashbox/ci/yaml/CompositeReaders.scala new file mode 100644 index 0000000..4df72e1 --- /dev/null +++ b/crashboxd/src/main/scala/io/crashbox/ci/yaml/CompositeReaders.scala @@ -0,0 +1,29 @@ +package io.crashbox.ci +package yaml + +trait CompositeReaders { + + implicit def mapReader[V: YamlReader] = new YamlReader[Map[String, V]] { + override def read(yml: YamlValue) = yml match { + case YamlMap(m) => + m.map { + case (key, value) => + key -> value.convertTo[V] + } + case YamlString.Empty => Map.empty[String, V] + case _ => formatError(yml, "mapping") + } + } + + implicit def seqReader[A: YamlReader] = new YamlReader[Seq[A]] { + override def read(yml: YamlValue) = yml match { + case YamlSeq(elements) => + elements.map { v => + v.convertTo[A] + } + case YamlString.Empty => Seq.empty[A] + case _ => formatError(yml, "sequence") + } + } + +} diff --git a/crashboxd/src/main/scala/io/crashbox/ci/yaml/DefaultReaders.scala b/crashboxd/src/main/scala/io/crashbox/ci/yaml/DefaultReaders.scala new file mode 100644 index 0000000..9d40bad --- /dev/null +++ b/crashboxd/src/main/scala/io/crashbox/ci/yaml/DefaultReaders.scala @@ -0,0 +1,6 @@ +package io.crashbox.ci +package yaml + +trait DefaultReaders extends SimpleReaders with CompositeReaders + +object DefaultReaders extends DefaultReaders diff --git a/crashboxd/src/main/scala/io/crashbox/ci/yaml/SimpleReaders.scala b/crashboxd/src/main/scala/io/crashbox/ci/yaml/SimpleReaders.scala new file mode 100644 index 0000000..df30a33 --- /dev/null +++ b/crashboxd/src/main/scala/io/crashbox/ci/yaml/SimpleReaders.scala @@ -0,0 +1,46 @@ +package io.crashbox.ci +package yaml + +import scala.util.Try + +trait SimpleReaders { + + class YamlStringReader[A](expected: String)(extract: String => Option[A]) + extends YamlReader[A] { + def read(yml: YamlValue) = yml match { + case YamlString(value) => + extract(value) match { + case Some(a) => a + case None => + throw new YamlFormatException( + s"""expected $expected, but found string type "$value"""") + } + case _ => formatError(yml, expected) + } + } + + implicit object valueReader extends YamlReader[YamlValue] { + def read(yaml: YamlValue) = yaml + } + + implicit object stringReader + extends YamlStringReader[String]("string")(s => Some(s)) + + implicit object byteReader + extends YamlStringReader[Byte]("byte")(s => Try { s.toByte }.toOption) + implicit object shortReader + extends YamlStringReader[Short]("short")(s => Try { s.toShort }.toOption) + implicit object intReader + extends YamlStringReader[Int]("integer")(s => Try { s.toInt }.toOption) + implicit object longReader + extends YamlStringReader[Long]("long")(s => Try { s.toLong }.toOption) + implicit object floatReader + extends YamlStringReader[Float]("float")(s => Try { s.toFloat }.toOption) + implicit object doubleReader + extends YamlStringReader[Double]("double")(s => + Try { s.toDouble }.toOption) + implicit object booleanReader + extends YamlStringReader[Boolean]("boolean")(s => + Try { s.toBoolean }.toOption) + +} diff --git a/crashboxd/src/main/scala/io/crashbox/ci/yaml/Yaml.scala b/crashboxd/src/main/scala/io/crashbox/ci/yaml/Yaml.scala new file mode 100644 index 0000000..0370c76 --- /dev/null +++ b/crashboxd/src/main/scala/io/crashbox/ci/yaml/Yaml.scala @@ -0,0 +1,36 @@ +package io.crashbox.ci +package yaml + +import java.util.{List => JList, Map => JMap} + +import scala.collection.JavaConverters._ + +import org.yaml.snakeyaml.{DumperOptions, Yaml => SYaml} +import org.yaml.snakeyaml.constructor.Constructor +import org.yaml.snakeyaml.representer.Representer +import org.yaml.snakeyaml.resolver.Resolver + +object Yaml { + + private def toYaml(yml: Any): YamlValue = yml match { + case m: JMap[_, _] => + YamlMap(m.asScala.toMap.map { case (k, v) => k.toString -> toYaml(v) }) + case l: JList[_] => YamlSeq(l.asScala.toList.map(toYaml(_))) + case s: String => YamlString(s) + case other => throw new YamlFormatException("Unknown YAML type: " + other) + } + + /** Strict parsing */ + def parse(data: String): YamlValue = { + val resolver = new Resolver { + override def addImplicitResolvers: Unit = {} + } + val yml = new SYaml(new Constructor(), + new Representer(), + new DumperOptions(), + resolver) + val node = yml.load(data) + toYaml(node) + } + +} diff --git a/crashboxd/src/main/scala/io/crashbox/ci/yaml/YamlFormatException.scala b/crashboxd/src/main/scala/io/crashbox/ci/yaml/YamlFormatException.scala new file mode 100644 index 0000000..1700900 --- /dev/null +++ b/crashboxd/src/main/scala/io/crashbox/ci/yaml/YamlFormatException.scala @@ -0,0 +1,4 @@ +package io.crashbox.ci +package yaml + +class YamlFormatException(message: String) extends RuntimeException(message) diff --git a/crashboxd/src/main/scala/io/crashbox/ci/yaml/YamlReader.scala b/crashboxd/src/main/scala/io/crashbox/ci/yaml/YamlReader.scala new file mode 100644 index 0000000..f486676 --- /dev/null +++ b/crashboxd/src/main/scala/io/crashbox/ci/yaml/YamlReader.scala @@ -0,0 +1,24 @@ +package io.crashbox.ci +package yaml + +trait YamlReader[A] { + + def read(yml: YamlValue): A + + protected def formatError(found: YamlValue, required: String) = { + val foundType = found match { + case _: YamlString => "string" + case _: YamlSeq => "sequence" + case _: YamlMap => "mapping" + } + + throw new YamlFormatException( + s"$found is of type $foundType, required: $required" + ) + } + + protected def readError(node: YamlValue, msg: String) = { + throw new YamlFormatException(node.toString + ": " + msg) + } + +} diff --git a/crashboxd/src/main/scala/io/crashbox/ci/yaml/values.scala b/crashboxd/src/main/scala/io/crashbox/ci/yaml/values.scala new file mode 100644 index 0000000..dd4fb0f --- /dev/null +++ b/crashboxd/src/main/scala/io/crashbox/ci/yaml/values.scala @@ -0,0 +1,12 @@ +package io.crashbox.ci +package yaml + +sealed trait YamlValue { + def convertTo[A: YamlReader]: A = implicitly[YamlReader[A]].read(this) +} +case class YamlString(value: String) extends YamlValue +object YamlString { + val Empty = YamlString("") +} +case class YamlMap(fields: Map[String, YamlValue]) extends YamlValue +case class YamlSeq(elements: Seq[YamlValue]) extends YamlValue -- cgit v1.2.3