Add GCS file storage implementation

2 files changed, 139 insertions, 4 deletions

diff --git a/src/main/scala/xyz/driver/core/file/GcsStorage.scala b/src/main/scala/xyz/driver/core/file/GcsStorage.scala
new file mode 100644
index 0000000..d0a3dad
--- /dev/null
+++ b/src/main/scala/xyz/driver/core/file/GcsStorage.scala
@@ -0,0 +1,72 @@
+package xyz.driver.core.file
+
+import java.io.{BufferedOutputStream, File, FileInputStream, FileOutputStream}
+import java.nio.file.{Path, Paths}
+
+import com.google.cloud.storage.Storage.BlobListOption
+import com.google.cloud.storage._
+import xyz.driver.core.time.Time
+import xyz.driver.core.{Name, Revision, generators}
+
+import scala.collection.JavaConverters._
+import scala.concurrent.{ExecutionContext, Future}
+import scalaz.{ListT, OptionT}
+
+class GcsStorage(storageClient: Storage, bucketName: Name[Bucket], executionContext: ExecutionContext)
+    extends FileStorage {
+  implicit private val execution: ExecutionContext = executionContext
+
+  override def upload(localSource: File, destination: Path): Future[Unit] = Future {
+    checkSafeFileName(destination) {
+      val blobId = BlobId.of(bucketName.value, destination.toString)
+      def acl    = Bucket.BlobWriteOption.predefinedAcl(Storage.PredefinedAcl.PUBLIC_READ)
+
+      storageClient.get(bucketName.value).create(blobId.getName, new FileInputStream(localSource), acl)
+    }
+  }
+
+  override def download(filePath: Path): OptionT[Future, File] = {
+    OptionT.optionT(Future {
+      Option(storageClient.get(bucketName.value, filePath.toString)).filterNot(_.getSize == 0).map {
+        blob =>
+          val tempDir             = System.getProperty("java.io.tmpdir")
+          val randomFolderName    = generators.nextUuid().toString
+          val tempDestinationFile = new File(Paths.get(tempDir, randomFolderName, filePath.toString).toString)
+
+          if (!tempDestinationFile.getParentFile.mkdirs()) {
+            throw new Exception(s"Failed to create temp directory to download file `$tempDestinationFile`")
+          } else {
+            val target = new BufferedOutputStream(new FileOutputStream(tempDestinationFile))
+            try target.write(blob.getContent())
+            finally target.close()
+            tempDestinationFile
+          }
+      }
+    })
+  }
+
+  override def delete(filePath: Path): Future[Unit] = Future {
+    storageClient.delete(BlobId.of(bucketName.value, filePath.toString))
+  }
+
+  override def list(path: Path): ListT[Future, FileLink] =
+    ListT.listT(Future {
+      val page = storageClient.list(
+        bucketName.value,
+        BlobListOption.currentDirectory(),
+        BlobListOption.prefix(path.toString)
+      )
+
+      page.iterateAll().asScala.map(blobToFileLink(path, _)).toList
+    })
+
+  protected def blobToFileLink(path: Path, blob: Blob): FileLink = {
+    FileLink(
+      Name(blob.getName),
+      Paths.get(path.toString, blob.getName),
+      Revision(blob.getGeneration.toString),
+      Time(blob.getUpdateTime),
+      blob.getSize
+    )
+  }
+}
diff --git a/src/test/scala/xyz/driver/core/FileTest.scala b/src/test/scala/xyz/driver/core/FileTest.scala
index a8379cf..a1b0329 100644
--- a/src/test/scala/xyz/driver/core/FileTest.scala
+++ b/src/test/scala/xyz/driver/core/FileTest.scala
@@ -1,15 +1,13 @@
 package xyz.driver.core
 
-import java.io.File
+import java.io.{File, FileInputStream}
 import java.nio.file.Paths
 
-import com.amazonaws.services.s3.AmazonS3
-import com.amazonaws.services.s3.model._
 import org.mockito.Matchers._
 import org.mockito.Mockito._
 import org.scalatest.mock.MockitoSugar
 import org.scalatest.{FlatSpec, Matchers}
-import xyz.driver.core.file.{FileSystemStorage, S3Storage}
+import xyz.driver.core.file.{FileSystemStorage, GcsStorage, S3Storage}
 
 import scala.concurrent.Await
 import scala.concurrent.duration._
@@ -17,6 +15,8 @@ import scala.concurrent.duration._
 class FileTest extends FlatSpec with Matchers with MockitoSugar {
 
   "S3 Storage" should "create and download local files and do other operations" in {
+    import com.amazonaws.services.s3.AmazonS3
+    import com.amazonaws.services.s3.model._
     import scala.collection.JavaConverters._
 
     val tempDir        = System.getProperty("java.io.tmpdir")
@@ -116,6 +116,69 @@ class FileTest extends FlatSpec with Matchers with MockitoSugar {
     filesAfterRemoval shouldBe empty
   }
 
+  "Google Cloud Storage" should "upload and download files" in {
+    import com.google.cloud.Page
+    import com.google.cloud.storage.{Blob, Bucket, Storage}
+    import Bucket.BlobWriteOption
+    import Storage.BlobListOption
+    import scala.collection.JavaConverters._
+
+    val tempDir        = System.getProperty("java.io.tmpdir")
+    val sourceTestFile = generateTestLocalFile(tempDir)
+    val testFileName   = "uploadTestFile"
+
+    val randomFolderName = java.util.UUID.randomUUID().toString
+    val testDirPath      = Paths.get(randomFolderName)
+    val testFilePath     = Paths.get(randomFolderName, testFileName)
+
+    val testBucket = Name[Bucket]("IamBucket")
+    val gcsMock    = mock[Storage]
+    val pageMock   = mock[Page[Blob]]
+    val bucketMock = mock[Bucket]
+    val blobMock   = mock[Blob]
+
+    when(blobMock.getName).thenReturn(testFileName)
+    when(blobMock.getGeneration).thenReturn(1000L)
+    when(blobMock.getUpdateTime).thenReturn(1493422254L)
+    when(blobMock.getSize).thenReturn(12345L)
+    when(blobMock.getContent()).thenReturn(Array[Byte](1, 2, 3))
+
+    val gcsStorage = new GcsStorage(gcsMock, testBucket, scala.concurrent.ExecutionContext.global)
+
+    when(pageMock.iterateAll()).thenReturn(
+      Iterator[Blob]().asJava,
+      Iterator[Blob](blobMock).asJava,
+      Iterator[Blob]().asJava
+    )
+    when(
+      gcsMock.list(testBucket.value, BlobListOption.currentDirectory(), BlobListOption.prefix(testDirPath.toString)))
+      .thenReturn(pageMock)
+
+    val filesBefore = Await.result(gcsStorage.list(testDirPath).run, 10 seconds)
+    filesBefore shouldBe empty
+
+    when(gcsMock.get(testBucket.value)).thenReturn(bucketMock)
+    when(gcsMock.get(testBucket.value, testFilePath.toString)).thenReturn(blobMock)
+    when(bucketMock.create(org.mockito.Matchers.eq(testFileName), any[FileInputStream], any[BlobWriteOption]))
+      .thenReturn(blobMock)
+
+    Await.result(gcsStorage.upload(sourceTestFile, testFilePath), 10 seconds)
+
+    val filesAfterUpload = Await.result(gcsStorage.list(testDirPath).run, 10 seconds)
+    filesAfterUpload.size should be(1)
+
+    val downloadedFile = Await.result(gcsStorage.download(testFilePath).run, 10 seconds)
+    downloadedFile shouldBe defined
+    downloadedFile.foreach {
+      _.getAbsolutePath should endWith(testFilePath.toString)
+    }
+
+    Await.result(gcsStorage.delete(testFilePath), 10 seconds)
+
+    val filesAfterRemoval = Await.result(gcsStorage.list(testDirPath).run, 10 seconds)
+    filesAfterRemoval shouldBe empty
+  }
+
   private def generateTestLocalFile(path: String): File = {
     val randomSourceFolderName = java.util.UUID.randomUUID().toString
     val sourceTestFile         = new File(Paths.get(path, randomSourceFolderName, "uploadTestFile").toString)