Add file existence checking to file storagev0.13.15

5 files changed, 56 insertions, 9 deletions

diff --git a/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala b/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala
index bfe6995..fab1307 100644
--- a/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala
+++ b/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala
@@ -1,7 +1,7 @@
 package xyz.driver.core.file
 
 import java.io.File
-import java.nio.file.{Path, Paths}
+import java.nio.file.{Files, Path, Paths}
 
 import xyz.driver.core.{Name, Revision}
 import xyz.driver.core.time.Time
@@ -12,7 +12,7 @@ import scalaz.{ListT, OptionT}
 class FileSystemStorage(executionContext: ExecutionContext) extends FileStorage {
   implicit private val execution = executionContext
 
-  def upload(localSource: File, destination: Path): Future[Unit] = Future {
+  override def upload(localSource: File, destination: Path): Future[Unit] = Future {
     checkSafeFileName(destination) {
       val destinationFile = destination.toFile
 
@@ -28,12 +28,12 @@ class FileSystemStorage(executionContext: ExecutionContext) extends FileStorage
     }
   }
 
-  def download(filePath: Path): OptionT[Future, File] =
+  override def download(filePath: Path): OptionT[Future, File] =
     OptionT.optionT(Future {
       Option(new File(filePath.toString)).filter(file => file.exists() && file.isFile)
     })
 
-  def delete(filePath: Path): Future[Unit] = Future {
+  override def delete(filePath: Path): Future[Unit] = Future {
     val file = new File(filePath.toString)
     if (file.delete()) ()
     else {
@@ -41,7 +41,7 @@ class FileSystemStorage(executionContext: ExecutionContext) extends FileStorage
     }
   }
 
-  def list(path: Path): ListT[Future, FileLink] =
+  override def list(path: Path): ListT[Future, FileLink] =
     ListT.listT(Future {
       val file = new File(path.toString)
       if (file.isDirectory) {
@@ -54,4 +54,9 @@ class FileSystemStorage(executionContext: ExecutionContext) extends FileStorage
         }
       } else List.empty[FileLink]
     })
+
+  override def exists(path: Path): Future[Boolean] = Future {
+    Files.exists(path)
+  }
+
 }
diff --git a/src/main/scala/xyz/driver/core/file/GcsStorage.scala b/src/main/scala/xyz/driver/core/file/GcsStorage.scala
index 0d8b918..deb8a0e 100644
--- a/src/main/scala/xyz/driver/core/file/GcsStorage.scala
+++ b/src/main/scala/xyz/driver/core/file/GcsStorage.scala
@@ -79,6 +79,15 @@ class GcsStorage(storageClient: Storage, bucketName: Name[Bucket], executionCont
     )
   }
 
+  override def exists(path: Path): Future[Boolean] = Future {
+    val blob = Option(
+      storageClient.get(
+        bucketName.value,
+        path.toString
+      ))
+    blob.isDefined
+  }
+
   override def signedFileUrl(filePath: Path, duration: Duration): OptionT[Future, URL] =
     OptionT.optionT(Future {
       Option(storageClient.get(bucketName.value, filePath.toString)).filterNot(_.getSize == 0).map { blob =>
diff --git a/src/main/scala/xyz/driver/core/file/S3Storage.scala b/src/main/scala/xyz/driver/core/file/S3Storage.scala
index 933b01a..7df3db2 100644
--- a/src/main/scala/xyz/driver/core/file/S3Storage.scala
+++ b/src/main/scala/xyz/driver/core/file/S3Storage.scala
@@ -15,13 +15,13 @@ import scalaz.{ListT, OptionT}
 class S3Storage(s3: AmazonS3, bucket: Name[Bucket], executionContext: ExecutionContext) extends FileStorage {
   implicit private val execution = executionContext
 
-  def upload(localSource: File, destination: Path): Future[Unit] = Future {
+  override def upload(localSource: File, destination: Path): Future[Unit] = Future {
     checkSafeFileName(destination) {
       val _ = s3.putObject(bucket.value, destination.toString, localSource).getETag
     }
   }
 
-  def download(filePath: Path): OptionT[Future, File] =
+  override def download(filePath: Path): OptionT[Future, File] =
     OptionT.optionT(Future {
       val tempDir             = System.getProperty("java.io.tmpdir")
       val randomFolderName    = randomUUID().toString
@@ -36,11 +36,11 @@ class S3Storage(s3: AmazonS3, bucket: Name[Bucket], executionContext: ExecutionC
       }
     })
 
-  def delete(filePath: Path): Future[Unit] = Future {
+  override def delete(filePath: Path): Future[Unit] = Future {
     s3.deleteObject(bucket.value, filePath.toString)
   }
 
-  def list(path: Path): ListT[Future, FileLink] =
+  override def list(path: Path): ListT[Future, FileLink] =
     ListT.listT(Future {
       import scala.collection.JavaConverters._
       val req = new ListObjectsV2Request().withBucketName(bucket.value).withPrefix(path.toString).withMaxKeys(2)
@@ -63,4 +63,9 @@ class S3Storage(s3: AmazonS3, bucket: Name[Bucket], executionContext: ExecutionC
         } filterNot isInSubFolder(path)
       } toList
     })
+
+  override def exists(path: Path): Future[Boolean] = Future {
+    s3.doesObjectExist(bucket.value, path.toString)
+  }
+
 }
diff --git a/src/main/scala/xyz/driver/core/file/package.scala b/src/main/scala/xyz/driver/core/file/package.scala
index 7203207..b2c679e 100644
--- a/src/main/scala/xyz/driver/core/file/package.scala
+++ b/src/main/scala/xyz/driver/core/file/package.scala
@@ -40,6 +40,8 @@ package file {
     /** List contents of a directory */
     def list(directoryPath: Path): ListT[Future, FileLink]
 
+    def exists(path: Path): Future[Boolean]
+
     /** List of characters to avoid in S3 (I would say file names in general)
       *
       * @see http://stackoverflow.com/questions/7116450/what-are-valid-s3-key-names-that-can-be-accessed-via-the-s3-rest-api
diff --git a/src/test/scala/xyz/driver/core/FileTest.scala b/src/test/scala/xyz/driver/core/FileTest.scala
index c35eb5b..246cd95 100644
--- a/src/test/scala/xyz/driver/core/FileTest.scala
+++ b/src/test/scala/xyz/driver/core/FileTest.scala
@@ -54,16 +54,25 @@ class FileTest extends FlatSpec with Matchers with MockitoSugar {
     when(amazonS3Mock.listObjectsV2(any[ListObjectsV2Request]())).thenReturn(s3ResultsMock)
     when(amazonS3Mock.putObject(testBucket.value, testFilePath.toString, sourceTestFile)).thenReturn(s3PutMock)
     when(amazonS3Mock.getObject(any[GetObjectRequest](), any[File]())).thenReturn(s3ObjectMetadataMock)
+    when(amazonS3Mock.doesObjectExist(testBucket.value, testFilePath.toString)).thenReturn(
+      false, // before file is uploaded
+      true // after file is uploaded
+    )
 
     val s3Storage = new S3Storage(amazonS3Mock, testBucket, scala.concurrent.ExecutionContext.global)
 
     val filesBefore = Await.result(s3Storage.list(testDirPath).run, 10 seconds)
     filesBefore shouldBe empty
 
+    val fileExistsBeforeUpload = Await.result(s3Storage.exists(testFilePath), 10 seconds)
+    fileExistsBeforeUpload should be(false)
+
     Await.result(s3Storage.upload(sourceTestFile, testFilePath), 10 seconds)
 
     val filesAfterUpload = Await.result(s3Storage.list(testDirPath).run, 10 seconds)
     filesAfterUpload.size should be(1)
+    val fileExistsAfterUpload = Await.result(s3Storage.exists(testFilePath), 10 seconds)
+    fileExistsAfterUpload should be(true)
     val uploadedFileLine = filesAfterUpload.head
     uploadedFileLine.name should be(Name[File](testFileName))
     uploadedFileLine.location should be(testFilePath)
@@ -96,10 +105,17 @@ class FileTest extends FlatSpec with Matchers with MockitoSugar {
     val filesBefore = Await.result(fileStorage.list(testDirPath).run, 10 seconds)
     filesBefore shouldBe empty
 
+    val fileExistsBeforeUpload = Await.result(fileStorage.exists(testFilePath), 10 seconds)
+    fileExistsBeforeUpload should be(false)
+
     Await.result(fileStorage.upload(sourceTestFile, testFilePath), 10 seconds)
 
     val filesAfterUpload = Await.result(fileStorage.list(testDirPath).run, 10 seconds)
     filesAfterUpload.size should be(1)
+
+    val fileExistsAfterUpload = Await.result(fileStorage.exists(testFilePath), 10 seconds)
+    fileExistsAfterUpload should be(true)
+
     val uploadedFileLine = filesAfterUpload.head
     uploadedFileLine.name should be(Name[File]("uploadTestFile"))
     uploadedFileLine.location should be(testFilePath)
@@ -152,10 +168,17 @@ class FileTest extends FlatSpec with Matchers with MockitoSugar {
     )
     when(gcsMock.list(testBucket.value, BlobListOption.currentDirectory(), BlobListOption.prefix(s"$testDirPath/")))
       .thenReturn(pageMock)
+    when(gcsMock.get(testBucket.value, testFilePath.toString)).thenReturn(
+      null, // before file is uploaded
+      blobMock // after file is uploaded
+    )
 
     val filesBefore = Await.result(gcsStorage.list(testDirPath).run, 10 seconds)
     filesBefore shouldBe empty
 
+    val fileExistsBeforeUpload = Await.result(gcsStorage.exists(testFilePath), 10 seconds)
+    fileExistsBeforeUpload should be(false)
+
     when(gcsMock.get(testBucket.value)).thenReturn(bucketMock)
     when(gcsMock.get(testBucket.value, testFilePath.toString)).thenReturn(blobMock)
     when(bucketMock.create(org.mockito.Matchers.eq(testFileName), any[FileInputStream], any[BlobWriteOption]))
@@ -166,6 +189,9 @@ class FileTest extends FlatSpec with Matchers with MockitoSugar {
     val filesAfterUpload = Await.result(gcsStorage.list(testDirPath).run, 10 seconds)
     filesAfterUpload.size should be(1)
 
+    val fileExistsAfterUpload = Await.result(gcsStorage.exists(testFilePath), 10 seconds)
+    fileExistsAfterUpload should be(true)
+
     val downloadedFile = Await.result(gcsStorage.download(testFilePath).run, 10 seconds)
     downloadedFile shouldBe defined
     downloadedFile.foreach {