Merge pull request #34 from drivergroup/zsmith/PDW-336-gcsv0.12.7

PDW-336 Add GCS file storage implementation

8 files changed, 332 insertions, 160 deletions

diff --git a/build.sbt b/build.sbt
index c393249..c4de456 100644
--- a/build.sbt
+++ b/build.sbt
@@ -15,6 +15,7 @@ lazy val core = (project in file("."))
     "org.mockito"                  % "mockito-core"          % "1.9.5" % "test",
     "com.github.swagger-akka-http" %% "swagger-akka-http"    % "0.9.1",
     "com.amazonaws"                % "aws-java-sdk-s3"       % "1.11.26",
+    "com.google.cloud"             % "google-cloud-storage"  % "0.9.4-beta",
     "com.typesafe.slick"           %% "slick"                % "3.1.1",
     "com.typesafe"                 % "config"                % "1.2.1",
     "com.typesafe.scala-logging"   %% "scala-logging"        % "3.4.0",
diff --git a/src/main/scala/xyz/driver/core/file.scala b/src/main/scala/xyz/driver/core/file.scala
deleted file mode 100644
index dcc4b87..0000000
--- a/src/main/scala/xyz/driver/core/file.scala
+++ /dev/null
@@ -1,155 +0,0 @@
-package xyz.driver.core
-
-import java.io.File
-import java.nio.file.{Path, Paths}
-import java.util.UUID._
-
-import com.amazonaws.services.s3.AmazonS3
-import com.amazonaws.services.s3.model.{Bucket, GetObjectRequest, ListObjectsV2Request}
-import xyz.driver.core.time.Time
-
-import scala.concurrent.{ExecutionContext, Future}
-import scalaz.{ListT, OptionT}
-
-object file {
-
-  final case class FileLink(
-          name: Name[File],
-          location: Path,
-          revision: Revision[File],
-          lastModificationDate: Time,
-          fileSize: Long
-  )
-
-  trait FileService {
-
-    def getFileLink(id: Name[File]): FileLink
-
-    def getFile(fileLink: FileLink): File
-  }
-
-  trait FileStorage {
-
-    def upload(localSource: File, destination: Path): Future[Unit]
-
-    def download(filePath: Path): OptionT[Future, File]
-
-    def delete(filePath: Path): Future[Unit]
-
-    def list(path: Path): ListT[Future, FileLink]
-
-    /** List of characters to avoid in S3 (I would say file names in general)
-      *
-      * @see http://stackoverflow.com/questions/7116450/what-are-valid-s3-key-names-that-can-be-accessed-via-the-s3-rest-api
-      */
-    private val illegalChars = "\\^`><{}][#%~|&@:,$=+?; "
-
-    protected def checkSafeFileName[T](filePath: Path)(f: => T): T = {
-      filePath.toString.find(c => illegalChars.contains(c)) match {
-        case Some(illegalCharacter) =>
-          throw new IllegalArgumentException(s"File name cannot contain character `$illegalCharacter`")
-        case None => f
-      }
-    }
-  }
-
-  class S3Storage(s3: AmazonS3, bucket: Name[Bucket], executionContext: ExecutionContext) extends FileStorage {
-    implicit private val execution = executionContext
-
-    def upload(localSource: File, destination: Path): Future[Unit] = Future {
-      checkSafeFileName(destination) {
-        val _ = s3.putObject(bucket.value, destination.toString, localSource).getETag
-      }
-    }
-
-    def download(filePath: Path): OptionT[Future, File] =
-      OptionT.optionT(Future {
-        val tempDir             = System.getProperty("java.io.tmpdir")
-        val randomFolderName    = randomUUID().toString
-        val tempDestinationFile = new File(Paths.get(tempDir, randomFolderName, filePath.toString).toString)
-
-        if (!tempDestinationFile.getParentFile.mkdirs()) {
-          throw new Exception(s"Failed to create temp directory to download file `$tempDestinationFile`")
-        } else {
-          Option(s3.getObject(new GetObjectRequest(bucket.value, filePath.toString), tempDestinationFile)).map { _ =>
-            tempDestinationFile
-          }
-        }
-      })
-
-    def delete(filePath: Path): Future[Unit] = Future {
-      s3.deleteObject(bucket.value, filePath.toString)
-    }
-
-    def list(path: Path): ListT[Future, FileLink] =
-      ListT.listT(Future {
-        import scala.collection.JavaConverters._
-        val req = new ListObjectsV2Request().withBucketName(bucket.value).withPrefix(path.toString).withMaxKeys(2)
-
-        def isInSubFolder(path: Path)(fileLink: FileLink) =
-          fileLink.location.toString.replace(path.toString + "/", "").contains("/")
-
-        Iterator.continually(s3.listObjectsV2(req)).takeWhile { result =>
-          req.setContinuationToken(result.getNextContinuationToken)
-          result.isTruncated
-        } flatMap { result =>
-          result.getObjectSummaries.asScala.toList.map { summary =>
-            FileLink(
-              Name[File](summary.getKey),
-              Paths.get(path.toString + "/" + summary.getKey),
-              Revision[File](summary.getETag),
-              Time(summary.getLastModified.getTime),
-              summary.getSize
-            )
-          } filterNot isInSubFolder(path)
-        } toList
-      })
-  }
-
-  class FileSystemStorage(executionContext: ExecutionContext) extends FileStorage {
-    implicit private val execution = executionContext
-
-    def upload(localSource: File, destination: Path): Future[Unit] = Future {
-      checkSafeFileName(destination) {
-        val destinationFile = destination.toFile
-
-        if (destinationFile.getParentFile.exists() || destinationFile.getParentFile.mkdirs()) {
-          if (localSource.renameTo(destinationFile)) ()
-          else {
-            throw new Exception(
-              s"Failed to move file from `${localSource.getCanonicalPath}` to `${destinationFile.getCanonicalPath}`")
-          }
-        } else {
-          throw new Exception(s"Failed to create parent directories for file `${destinationFile.getCanonicalPath}`")
-        }
-      }
-    }
-
-    def download(filePath: Path): OptionT[Future, File] =
-      OptionT.optionT(Future {
-        Option(new File(filePath.toString)).filter(file => file.exists() && file.isFile)
-      })
-
-    def delete(filePath: Path): Future[Unit] = Future {
-      val file = new File(filePath.toString)
-      if (file.delete()) ()
-      else {
-        throw new Exception(s"Failed to delete file $file" + (if (!file.exists()) ", file does not exist." else "."))
-      }
-    }
-
-    def list(path: Path): ListT[Future, FileLink] =
-      ListT.listT(Future {
-        val file = new File(path.toString)
-        if (file.isDirectory) {
-          file.listFiles().toList.filter(_.isFile).map { file =>
-            FileLink(Name[File](file.getName),
-                     Paths.get(file.getPath),
-                     Revision[File](file.hashCode.toString),
-                     Time(file.lastModified()),
-                     file.length())
-          }
-        } else List.empty[FileLink]
-      })
-  }
-}
diff --git a/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala b/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala
new file mode 100644
index 0000000..bfe6995
--- /dev/null
+++ b/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala
@@ -0,0 +1,57 @@
+package xyz.driver.core.file
+
+import java.io.File
+import java.nio.file.{Path, Paths}
+
+import xyz.driver.core.{Name, Revision}
+import xyz.driver.core.time.Time
+
+import scala.concurrent.{ExecutionContext, Future}
+import scalaz.{ListT, OptionT}
+
+class FileSystemStorage(executionContext: ExecutionContext) extends FileStorage {
+  implicit private val execution = executionContext
+
+  def upload(localSource: File, destination: Path): Future[Unit] = Future {
+    checkSafeFileName(destination) {
+      val destinationFile = destination.toFile
+
+      if (destinationFile.getParentFile.exists() || destinationFile.getParentFile.mkdirs()) {
+        if (localSource.renameTo(destinationFile)) ()
+        else {
+          throw new Exception(
+            s"Failed to move file from `${localSource.getCanonicalPath}` to `${destinationFile.getCanonicalPath}`")
+        }
+      } else {
+        throw new Exception(s"Failed to create parent directories for file `${destinationFile.getCanonicalPath}`")
+      }
+    }
+  }
+
+  def download(filePath: Path): OptionT[Future, File] =
+    OptionT.optionT(Future {
+      Option(new File(filePath.toString)).filter(file => file.exists() && file.isFile)
+    })
+
+  def delete(filePath: Path): Future[Unit] = Future {
+    val file = new File(filePath.toString)
+    if (file.delete()) ()
+    else {
+      throw new Exception(s"Failed to delete file $file" + (if (!file.exists()) ", file does not exist." else "."))
+    }
+  }
+
+  def list(path: Path): ListT[Future, FileLink] =
+    ListT.listT(Future {
+      val file = new File(path.toString)
+      if (file.isDirectory) {
+        file.listFiles().toList.filter(_.isFile).map { file =>
+          FileLink(Name[File](file.getName),
+                   Paths.get(file.getPath),
+                   Revision[File](file.hashCode.toString),
+                   Time(file.lastModified()),
+                   file.length())
+        }
+      } else List.empty[FileLink]
+    })
+}
diff --git a/src/main/scala/xyz/driver/core/file/GcsStorage.scala b/src/main/scala/xyz/driver/core/file/GcsStorage.scala
new file mode 100644
index 0000000..6c2746e
--- /dev/null
+++ b/src/main/scala/xyz/driver/core/file/GcsStorage.scala
@@ -0,0 +1,82 @@
+package xyz.driver.core.file
+
+import java.io.{BufferedOutputStream, File, FileInputStream, FileOutputStream}
+import java.net.URL
+import java.nio.file.{Path, Paths}
+import java.util.concurrent.TimeUnit
+
+import com.google.cloud.storage.Storage.BlobListOption
+import com.google.cloud.storage._
+import xyz.driver.core.time.Time
+import xyz.driver.core.{Name, Revision, generators}
+
+import scala.collection.JavaConverters._
+import scala.concurrent.duration.Duration
+import scala.concurrent.{ExecutionContext, Future}
+import scalaz.{ListT, OptionT}
+
+class GcsStorage(storageClient: Storage, bucketName: Name[Bucket], executionContext: ExecutionContext)
+    extends SignedFileStorage {
+  implicit private val execution: ExecutionContext = executionContext
+
+  override def upload(localSource: File, destination: Path): Future[Unit] = Future {
+    checkSafeFileName(destination) {
+      val blobId = BlobId.of(bucketName.value, destination.toString)
+      def acl    = Bucket.BlobWriteOption.predefinedAcl(Storage.PredefinedAcl.PUBLIC_READ)
+
+      storageClient.get(bucketName.value).create(blobId.getName, new FileInputStream(localSource), acl)
+    }
+  }
+
+  override def download(filePath: Path): OptionT[Future, File] = {
+    OptionT.optionT(Future {
+      Option(storageClient.get(bucketName.value, filePath.toString)).filterNot(_.getSize == 0).map {
+        blob =>
+          val tempDir             = System.getProperty("java.io.tmpdir")
+          val randomFolderName    = generators.nextUuid().toString
+          val tempDestinationFile = new File(Paths.get(tempDir, randomFolderName, filePath.toString).toString)
+
+          if (!tempDestinationFile.getParentFile.mkdirs()) {
+            throw new Exception(s"Failed to create temp directory to download file `$tempDestinationFile`")
+          } else {
+            val target = new BufferedOutputStream(new FileOutputStream(tempDestinationFile))
+            try target.write(blob.getContent())
+            finally target.close()
+            tempDestinationFile
+          }
+      }
+    })
+  }
+
+  override def delete(filePath: Path): Future[Unit] = Future {
+    storageClient.delete(BlobId.of(bucketName.value, filePath.toString))
+  }
+
+  override def list(path: Path): ListT[Future, FileLink] =
+    ListT.listT(Future {
+      val page = storageClient.list(
+        bucketName.value,
+        BlobListOption.currentDirectory(),
+        BlobListOption.prefix(path.toString)
+      )
+
+      page.iterateAll().asScala.map(blobToFileLink(path, _)).toList
+    })
+
+  protected def blobToFileLink(path: Path, blob: Blob): FileLink = {
+    FileLink(
+      Name(blob.getName),
+      Paths.get(path.toString, blob.getName),
+      Revision(blob.getGeneration.toString),
+      Time(blob.getUpdateTime),
+      blob.getSize
+    )
+  }
+
+  override def signedFileUrl(filePath: Path, duration: Duration): OptionT[Future, URL] =
+    OptionT.optionT(Future {
+      Option(storageClient.get(bucketName.value, filePath.toString)).filterNot(_.getSize == 0).map { blob =>
+        blob.signUrl(duration.toSeconds, TimeUnit.SECONDS)
+      }
+    })
+}
diff --git a/src/main/scala/xyz/driver/core/file/S3Storage.scala b/src/main/scala/xyz/driver/core/file/S3Storage.scala
new file mode 100644
index 0000000..50bfe85
--- /dev/null
+++ b/src/main/scala/xyz/driver/core/file/S3Storage.scala
@@ -0,0 +1,64 @@
+package xyz.driver.core.file
+
+import java.io.File
+import java.nio.file.{Path, Paths}
+import java.util.UUID.randomUUID
+
+import com.amazonaws.services.s3.AmazonS3
+import com.amazonaws.services.s3.model.{Bucket, GetObjectRequest, ListObjectsV2Request}
+import xyz.driver.core.{Name, Revision}
+import xyz.driver.core.time.Time
+
+import scala.concurrent.{ExecutionContext, Future}
+import scalaz.{ListT, OptionT}
+
+class S3Storage(s3: AmazonS3, bucket: Name[Bucket], executionContext: ExecutionContext) extends FileStorage {
+  implicit private val execution = executionContext
+
+  def upload(localSource: File, destination: Path): Future[Unit] = Future {
+    checkSafeFileName(destination) {
+      val _ = s3.putObject(bucket.value, destination.toString, localSource).getETag
+    }
+  }
+
+  def download(filePath: Path): OptionT[Future, File] =
+    OptionT.optionT(Future {
+      val tempDir             = System.getProperty("java.io.tmpdir")
+      val randomFolderName    = randomUUID().toString
+      val tempDestinationFile = new File(Paths.get(tempDir, randomFolderName, filePath.toString).toString)
+
+      if (!tempDestinationFile.getParentFile.mkdirs()) {
+        throw new Exception(s"Failed to create temp directory to download file `$tempDestinationFile`")
+      } else {
+        Option(s3.getObject(new GetObjectRequest(bucket.value, filePath.toString), tempDestinationFile)).map { _ =>
+          tempDestinationFile
+        }
+      }
+    })
+
+  def delete(filePath: Path): Future[Unit] = Future {
+    s3.deleteObject(bucket.value, filePath.toString)
+  }
+
+  def list(path: Path): ListT[Future, FileLink] =
+    ListT.listT(Future {
+      import scala.collection.JavaConverters._
+      val req = new ListObjectsV2Request().withBucketName(bucket.value).withPrefix(path.toString).withMaxKeys(2)
+
+      def isInSubFolder(path: Path)(fileLink: FileLink) =
+        fileLink.location.toString.replace(path.toString + "/", "").contains("/")
+
+      Iterator.continually(s3.listObjectsV2(req)).takeWhile { result =>
+        req.setContinuationToken(result.getNextContinuationToken)
+        result.isTruncated
+      } flatMap { result =>
+        result.getObjectSummaries.asScala.toList.map { summary =>
+          FileLink(Name[File](summary.getKey),
+                   Paths.get(path.toString + "/" + summary.getKey),
+                   Revision[File](summary.getETag),
+                   Time(summary.getLastModified.getTime),
+                   summary.getSize)
+        } filterNot isInSubFolder(path)
+      } toList
+    })
+}
diff --git a/src/main/scala/xyz/driver/core/file/package.scala b/src/main/scala/xyz/driver/core/file/package.scala
new file mode 100644
index 0000000..9000894
--- /dev/null
+++ b/src/main/scala/xyz/driver/core/file/package.scala
@@ -0,0 +1,60 @@
+package xyz.driver.core
+
+import java.io.File
+import java.nio.file.Path
+
+import xyz.driver.core.time.Time
+
+import scala.concurrent.Future
+import scalaz.{ListT, OptionT}
+
+package file {
+
+  import java.net.URL
+
+  import scala.concurrent.duration.Duration
+
+  final case class FileLink(
+          name: Name[File],
+          location: Path,
+          revision: Revision[File],
+          lastModificationDate: Time,
+          fileSize: Long
+  )
+
+  trait FileService {
+
+    def getFileLink(id: Name[File]): FileLink
+
+    def getFile(fileLink: FileLink): File
+  }
+
+  trait FileStorage {
+
+    def upload(localSource: File, destination: Path): Future[Unit]
+
+    def download(filePath: Path): OptionT[Future, File]
+
+    def delete(filePath: Path): Future[Unit]
+
+    def list(path: Path): ListT[Future, FileLink]
+
+    /** List of characters to avoid in S3 (I would say file names in general)
+      *
+      * @see http://stackoverflow.com/questions/7116450/what-are-valid-s3-key-names-that-can-be-accessed-via-the-s3-rest-api
+      */
+    private val illegalChars = "\\^`><{}][#%~|&@:,$=+?; "
+
+    protected def checkSafeFileName[T](filePath: Path)(f: => T): T = {
+      filePath.toString.find(c => illegalChars.contains(c)) match {
+        case Some(illegalCharacter) =>
+          throw new IllegalArgumentException(s"File name cannot contain character `$illegalCharacter`")
+        case None => f
+      }
+    }
+  }
+
+  trait SignedFileStorage extends FileStorage {
+    def signedFileUrl(filePath: Path, duration: Duration): OptionT[Future, URL]
+  }
+}
diff --git a/src/test/scala/xyz/driver/core/DateTest.scala b/src/test/scala/xyz/driver/core/DateTest.scala
index c1185cd..0cf8a9e 100644
--- a/src/test/scala/xyz/driver/core/DateTest.scala
+++ b/src/test/scala/xyz/driver/core/DateTest.scala
@@ -38,7 +38,7 @@ class DateTest extends FlatSpec with Matchers with Checkers {
         case Seq(a, b) =>
           if (a.year == b.year) {
             if (a.month == b.month) {
-              a.day < b.day
+              a.day <= b.day
             } else {
               a.month < b.month
             }
diff --git a/src/test/scala/xyz/driver/core/FileTest.scala b/src/test/scala/xyz/driver/core/FileTest.scala
index a8379cf..a1b0329 100644
--- a/src/test/scala/xyz/driver/core/FileTest.scala
+++ b/src/test/scala/xyz/driver/core/FileTest.scala
@@ -1,15 +1,13 @@
 package xyz.driver.core
 
-import java.io.File
+import java.io.{File, FileInputStream}
 import java.nio.file.Paths
 
-import com.amazonaws.services.s3.AmazonS3
-import com.amazonaws.services.s3.model._
 import org.mockito.Matchers._
 import org.mockito.Mockito._
 import org.scalatest.mock.MockitoSugar
 import org.scalatest.{FlatSpec, Matchers}
-import xyz.driver.core.file.{FileSystemStorage, S3Storage}
+import xyz.driver.core.file.{FileSystemStorage, GcsStorage, S3Storage}
 
 import scala.concurrent.Await
 import scala.concurrent.duration._
@@ -17,6 +15,8 @@ import scala.concurrent.duration._
 class FileTest extends FlatSpec with Matchers with MockitoSugar {
 
   "S3 Storage" should "create and download local files and do other operations" in {
+    import com.amazonaws.services.s3.AmazonS3
+    import com.amazonaws.services.s3.model._
     import scala.collection.JavaConverters._
 
     val tempDir        = System.getProperty("java.io.tmpdir")
@@ -116,6 +116,69 @@ class FileTest extends FlatSpec with Matchers with MockitoSugar {
     filesAfterRemoval shouldBe empty
   }
 
+  "Google Cloud Storage" should "upload and download files" in {
+    import com.google.cloud.Page
+    import com.google.cloud.storage.{Blob, Bucket, Storage}
+    import Bucket.BlobWriteOption
+    import Storage.BlobListOption
+    import scala.collection.JavaConverters._
+
+    val tempDir        = System.getProperty("java.io.tmpdir")
+    val sourceTestFile = generateTestLocalFile(tempDir)
+    val testFileName   = "uploadTestFile"
+
+    val randomFolderName = java.util.UUID.randomUUID().toString
+    val testDirPath      = Paths.get(randomFolderName)
+    val testFilePath     = Paths.get(randomFolderName, testFileName)
+
+    val testBucket = Name[Bucket]("IamBucket")
+    val gcsMock    = mock[Storage]
+    val pageMock   = mock[Page[Blob]]
+    val bucketMock = mock[Bucket]
+    val blobMock   = mock[Blob]
+
+    when(blobMock.getName).thenReturn(testFileName)
+    when(blobMock.getGeneration).thenReturn(1000L)
+    when(blobMock.getUpdateTime).thenReturn(1493422254L)
+    when(blobMock.getSize).thenReturn(12345L)
+    when(blobMock.getContent()).thenReturn(Array[Byte](1, 2, 3))
+
+    val gcsStorage = new GcsStorage(gcsMock, testBucket, scala.concurrent.ExecutionContext.global)
+
+    when(pageMock.iterateAll()).thenReturn(
+      Iterator[Blob]().asJava,
+      Iterator[Blob](blobMock).asJava,
+      Iterator[Blob]().asJava
+    )
+    when(
+      gcsMock.list(testBucket.value, BlobListOption.currentDirectory(), BlobListOption.prefix(testDirPath.toString)))
+      .thenReturn(pageMock)
+
+    val filesBefore = Await.result(gcsStorage.list(testDirPath).run, 10 seconds)
+    filesBefore shouldBe empty
+
+    when(gcsMock.get(testBucket.value)).thenReturn(bucketMock)
+    when(gcsMock.get(testBucket.value, testFilePath.toString)).thenReturn(blobMock)
+    when(bucketMock.create(org.mockito.Matchers.eq(testFileName), any[FileInputStream], any[BlobWriteOption]))
+      .thenReturn(blobMock)
+
+    Await.result(gcsStorage.upload(sourceTestFile, testFilePath), 10 seconds)
+
+    val filesAfterUpload = Await.result(gcsStorage.list(testDirPath).run, 10 seconds)
+    filesAfterUpload.size should be(1)
+
+    val downloadedFile = Await.result(gcsStorage.download(testFilePath).run, 10 seconds)
+    downloadedFile shouldBe defined
+    downloadedFile.foreach {
+      _.getAbsolutePath should endWith(testFilePath.toString)
+    }
+
+    Await.result(gcsStorage.delete(testFilePath), 10 seconds)
+
+    val filesAfterRemoval = Await.result(gcsStorage.list(testDirPath).run, 10 seconds)
+    filesAfterRemoval shouldBe empty
+  }
+
   private def generateTestLocalFile(path: String): File = {
     val randomSourceFolderName = java.util.UUID.randomUUID().toString
     val sourceTestFile         = new File(Paths.get(path, randomSourceFolderName, "uploadTestFile").toString)