diff options
author | Zach Smith <zach@driver.xyz> | 2017-04-28 16:43:02 -0700 |
---|---|---|
committer | Zach Smith <zach@driver.xyz> | 2017-05-04 10:34:33 -0700 |
commit | 4eef6fc976117788527e0e65acd2068d0e65893f (patch) | |
tree | a380596598e5997cc65f5b44a54ce64f4c5f23ec /src | |
parent | e9ea2ab850a5251369217b48fd99b65ef4b6985d (diff) | |
download | driver-core-4eef6fc976117788527e0e65acd2068d0e65893f.tar.gz driver-core-4eef6fc976117788527e0e65acd2068d0e65893f.tar.bz2 driver-core-4eef6fc976117788527e0e65acd2068d0e65893f.zip |
Add GCS file storage implementation
Diffstat (limited to 'src')
-rw-r--r-- | src/main/scala/xyz/driver/core/file/GcsStorage.scala | 72 | ||||
-rw-r--r-- | src/test/scala/xyz/driver/core/FileTest.scala | 71 |
2 files changed, 139 insertions, 4 deletions
diff --git a/src/main/scala/xyz/driver/core/file/GcsStorage.scala b/src/main/scala/xyz/driver/core/file/GcsStorage.scala new file mode 100644 index 0000000..d0a3dad --- /dev/null +++ b/src/main/scala/xyz/driver/core/file/GcsStorage.scala @@ -0,0 +1,72 @@ +package xyz.driver.core.file + +import java.io.{BufferedOutputStream, File, FileInputStream, FileOutputStream} +import java.nio.file.{Path, Paths} + +import com.google.cloud.storage.Storage.BlobListOption +import com.google.cloud.storage._ +import xyz.driver.core.time.Time +import xyz.driver.core.{Name, Revision, generators} + +import scala.collection.JavaConverters._ +import scala.concurrent.{ExecutionContext, Future} +import scalaz.{ListT, OptionT} + +class GcsStorage(storageClient: Storage, bucketName: Name[Bucket], executionContext: ExecutionContext) + extends FileStorage { + implicit private val execution: ExecutionContext = executionContext + + override def upload(localSource: File, destination: Path): Future[Unit] = Future { + checkSafeFileName(destination) { + val blobId = BlobId.of(bucketName.value, destination.toString) + def acl = Bucket.BlobWriteOption.predefinedAcl(Storage.PredefinedAcl.PUBLIC_READ) + + storageClient.get(bucketName.value).create(blobId.getName, new FileInputStream(localSource), acl) + } + } + + override def download(filePath: Path): OptionT[Future, File] = { + OptionT.optionT(Future { + Option(storageClient.get(bucketName.value, filePath.toString)).filterNot(_.getSize == 0).map { + blob => + val tempDir = System.getProperty("java.io.tmpdir") + val randomFolderName = generators.nextUuid().toString + val tempDestinationFile = new File(Paths.get(tempDir, randomFolderName, filePath.toString).toString) + + if (!tempDestinationFile.getParentFile.mkdirs()) { + throw new Exception(s"Failed to create temp directory to download file `$tempDestinationFile`") + } else { + val target = new BufferedOutputStream(new FileOutputStream(tempDestinationFile)) + try target.write(blob.getContent()) + finally target.close() + tempDestinationFile + } + } + }) + } + + override def delete(filePath: Path): Future[Unit] = Future { + storageClient.delete(BlobId.of(bucketName.value, filePath.toString)) + } + + override def list(path: Path): ListT[Future, FileLink] = + ListT.listT(Future { + val page = storageClient.list( + bucketName.value, + BlobListOption.currentDirectory(), + BlobListOption.prefix(path.toString) + ) + + page.iterateAll().asScala.map(blobToFileLink(path, _)).toList + }) + + protected def blobToFileLink(path: Path, blob: Blob): FileLink = { + FileLink( + Name(blob.getName), + Paths.get(path.toString, blob.getName), + Revision(blob.getGeneration.toString), + Time(blob.getUpdateTime), + blob.getSize + ) + } +} diff --git a/src/test/scala/xyz/driver/core/FileTest.scala b/src/test/scala/xyz/driver/core/FileTest.scala index a8379cf..a1b0329 100644 --- a/src/test/scala/xyz/driver/core/FileTest.scala +++ b/src/test/scala/xyz/driver/core/FileTest.scala @@ -1,15 +1,13 @@ package xyz.driver.core -import java.io.File +import java.io.{File, FileInputStream} import java.nio.file.Paths -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model._ import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalatest.mock.MockitoSugar import org.scalatest.{FlatSpec, Matchers} -import xyz.driver.core.file.{FileSystemStorage, S3Storage} +import xyz.driver.core.file.{FileSystemStorage, GcsStorage, S3Storage} import scala.concurrent.Await import scala.concurrent.duration._ @@ -17,6 +15,8 @@ import scala.concurrent.duration._ class FileTest extends FlatSpec with Matchers with MockitoSugar { "S3 Storage" should "create and download local files and do other operations" in { + import com.amazonaws.services.s3.AmazonS3 + import com.amazonaws.services.s3.model._ import scala.collection.JavaConverters._ val tempDir = System.getProperty("java.io.tmpdir") @@ -116,6 +116,69 @@ class FileTest extends FlatSpec with Matchers with MockitoSugar { filesAfterRemoval shouldBe empty } + "Google Cloud Storage" should "upload and download files" in { + import com.google.cloud.Page + import com.google.cloud.storage.{Blob, Bucket, Storage} + import Bucket.BlobWriteOption + import Storage.BlobListOption + import scala.collection.JavaConverters._ + + val tempDir = System.getProperty("java.io.tmpdir") + val sourceTestFile = generateTestLocalFile(tempDir) + val testFileName = "uploadTestFile" + + val randomFolderName = java.util.UUID.randomUUID().toString + val testDirPath = Paths.get(randomFolderName) + val testFilePath = Paths.get(randomFolderName, testFileName) + + val testBucket = Name[Bucket]("IamBucket") + val gcsMock = mock[Storage] + val pageMock = mock[Page[Blob]] + val bucketMock = mock[Bucket] + val blobMock = mock[Blob] + + when(blobMock.getName).thenReturn(testFileName) + when(blobMock.getGeneration).thenReturn(1000L) + when(blobMock.getUpdateTime).thenReturn(1493422254L) + when(blobMock.getSize).thenReturn(12345L) + when(blobMock.getContent()).thenReturn(Array[Byte](1, 2, 3)) + + val gcsStorage = new GcsStorage(gcsMock, testBucket, scala.concurrent.ExecutionContext.global) + + when(pageMock.iterateAll()).thenReturn( + Iterator[Blob]().asJava, + Iterator[Blob](blobMock).asJava, + Iterator[Blob]().asJava + ) + when( + gcsMock.list(testBucket.value, BlobListOption.currentDirectory(), BlobListOption.prefix(testDirPath.toString))) + .thenReturn(pageMock) + + val filesBefore = Await.result(gcsStorage.list(testDirPath).run, 10 seconds) + filesBefore shouldBe empty + + when(gcsMock.get(testBucket.value)).thenReturn(bucketMock) + when(gcsMock.get(testBucket.value, testFilePath.toString)).thenReturn(blobMock) + when(bucketMock.create(org.mockito.Matchers.eq(testFileName), any[FileInputStream], any[BlobWriteOption])) + .thenReturn(blobMock) + + Await.result(gcsStorage.upload(sourceTestFile, testFilePath), 10 seconds) + + val filesAfterUpload = Await.result(gcsStorage.list(testDirPath).run, 10 seconds) + filesAfterUpload.size should be(1) + + val downloadedFile = Await.result(gcsStorage.download(testFilePath).run, 10 seconds) + downloadedFile shouldBe defined + downloadedFile.foreach { + _.getAbsolutePath should endWith(testFilePath.toString) + } + + Await.result(gcsStorage.delete(testFilePath), 10 seconds) + + val filesAfterRemoval = Await.result(gcsStorage.list(testDirPath).run, 10 seconds) + filesAfterRemoval shouldBe empty + } + private def generateTestLocalFile(path: String): File = { val randomSourceFolderName = java.util.UUID.randomUUID().toString val sourceTestFile = new File(Paths.get(path, randomSourceFolderName, "uploadTestFile").toString) |