aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZach Smith <zach@driver.xyz>2017-04-27 11:55:16 -0700
committerZach Smith <zach@driver.xyz>2017-05-04 10:34:33 -0700
commite9ea2ab850a5251369217b48fd99b65ef4b6985d (patch)
treeede2f324d0a4d875e19187ca789dc8105817aaf2
parentd6695390a60aaaa907f0b8744ce5757fde30a9eb (diff)
downloaddriver-core-e9ea2ab850a5251369217b48fd99b65ef4b6985d.tar.gz
driver-core-e9ea2ab850a5251369217b48fd99b65ef4b6985d.tar.bz2
driver-core-e9ea2ab850a5251369217b48fd99b65ef4b6985d.zip
Split FileStorage implementations into separate files
-rw-r--r--src/main/scala/xyz/driver/core/file.scala155
-rw-r--r--src/main/scala/xyz/driver/core/file/FileSystemStorage.scala57
-rw-r--r--src/main/scala/xyz/driver/core/file/S3Storage.scala64
-rw-r--r--src/main/scala/xyz/driver/core/file/package.scala51
4 files changed, 172 insertions, 155 deletions
diff --git a/src/main/scala/xyz/driver/core/file.scala b/src/main/scala/xyz/driver/core/file.scala
deleted file mode 100644
index dcc4b87..0000000
--- a/src/main/scala/xyz/driver/core/file.scala
+++ /dev/null
@@ -1,155 +0,0 @@
-package xyz.driver.core
-
-import java.io.File
-import java.nio.file.{Path, Paths}
-import java.util.UUID._
-
-import com.amazonaws.services.s3.AmazonS3
-import com.amazonaws.services.s3.model.{Bucket, GetObjectRequest, ListObjectsV2Request}
-import xyz.driver.core.time.Time
-
-import scala.concurrent.{ExecutionContext, Future}
-import scalaz.{ListT, OptionT}
-
-object file {
-
- final case class FileLink(
- name: Name[File],
- location: Path,
- revision: Revision[File],
- lastModificationDate: Time,
- fileSize: Long
- )
-
- trait FileService {
-
- def getFileLink(id: Name[File]): FileLink
-
- def getFile(fileLink: FileLink): File
- }
-
- trait FileStorage {
-
- def upload(localSource: File, destination: Path): Future[Unit]
-
- def download(filePath: Path): OptionT[Future, File]
-
- def delete(filePath: Path): Future[Unit]
-
- def list(path: Path): ListT[Future, FileLink]
-
- /** List of characters to avoid in S3 (I would say file names in general)
- *
- * @see http://stackoverflow.com/questions/7116450/what-are-valid-s3-key-names-that-can-be-accessed-via-the-s3-rest-api
- */
- private val illegalChars = "\\^`><{}][#%~|&@:,$=+?; "
-
- protected def checkSafeFileName[T](filePath: Path)(f: => T): T = {
- filePath.toString.find(c => illegalChars.contains(c)) match {
- case Some(illegalCharacter) =>
- throw new IllegalArgumentException(s"File name cannot contain character `$illegalCharacter`")
- case None => f
- }
- }
- }
-
- class S3Storage(s3: AmazonS3, bucket: Name[Bucket], executionContext: ExecutionContext) extends FileStorage {
- implicit private val execution = executionContext
-
- def upload(localSource: File, destination: Path): Future[Unit] = Future {
- checkSafeFileName(destination) {
- val _ = s3.putObject(bucket.value, destination.toString, localSource).getETag
- }
- }
-
- def download(filePath: Path): OptionT[Future, File] =
- OptionT.optionT(Future {
- val tempDir = System.getProperty("java.io.tmpdir")
- val randomFolderName = randomUUID().toString
- val tempDestinationFile = new File(Paths.get(tempDir, randomFolderName, filePath.toString).toString)
-
- if (!tempDestinationFile.getParentFile.mkdirs()) {
- throw new Exception(s"Failed to create temp directory to download file `$tempDestinationFile`")
- } else {
- Option(s3.getObject(new GetObjectRequest(bucket.value, filePath.toString), tempDestinationFile)).map { _ =>
- tempDestinationFile
- }
- }
- })
-
- def delete(filePath: Path): Future[Unit] = Future {
- s3.deleteObject(bucket.value, filePath.toString)
- }
-
- def list(path: Path): ListT[Future, FileLink] =
- ListT.listT(Future {
- import scala.collection.JavaConverters._
- val req = new ListObjectsV2Request().withBucketName(bucket.value).withPrefix(path.toString).withMaxKeys(2)
-
- def isInSubFolder(path: Path)(fileLink: FileLink) =
- fileLink.location.toString.replace(path.toString + "/", "").contains("/")
-
- Iterator.continually(s3.listObjectsV2(req)).takeWhile { result =>
- req.setContinuationToken(result.getNextContinuationToken)
- result.isTruncated
- } flatMap { result =>
- result.getObjectSummaries.asScala.toList.map { summary =>
- FileLink(
- Name[File](summary.getKey),
- Paths.get(path.toString + "/" + summary.getKey),
- Revision[File](summary.getETag),
- Time(summary.getLastModified.getTime),
- summary.getSize
- )
- } filterNot isInSubFolder(path)
- } toList
- })
- }
-
- class FileSystemStorage(executionContext: ExecutionContext) extends FileStorage {
- implicit private val execution = executionContext
-
- def upload(localSource: File, destination: Path): Future[Unit] = Future {
- checkSafeFileName(destination) {
- val destinationFile = destination.toFile
-
- if (destinationFile.getParentFile.exists() || destinationFile.getParentFile.mkdirs()) {
- if (localSource.renameTo(destinationFile)) ()
- else {
- throw new Exception(
- s"Failed to move file from `${localSource.getCanonicalPath}` to `${destinationFile.getCanonicalPath}`")
- }
- } else {
- throw new Exception(s"Failed to create parent directories for file `${destinationFile.getCanonicalPath}`")
- }
- }
- }
-
- def download(filePath: Path): OptionT[Future, File] =
- OptionT.optionT(Future {
- Option(new File(filePath.toString)).filter(file => file.exists() && file.isFile)
- })
-
- def delete(filePath: Path): Future[Unit] = Future {
- val file = new File(filePath.toString)
- if (file.delete()) ()
- else {
- throw new Exception(s"Failed to delete file $file" + (if (!file.exists()) ", file does not exist." else "."))
- }
- }
-
- def list(path: Path): ListT[Future, FileLink] =
- ListT.listT(Future {
- val file = new File(path.toString)
- if (file.isDirectory) {
- file.listFiles().toList.filter(_.isFile).map { file =>
- FileLink(Name[File](file.getName),
- Paths.get(file.getPath),
- Revision[File](file.hashCode.toString),
- Time(file.lastModified()),
- file.length())
- }
- } else List.empty[FileLink]
- })
- }
-}
diff --git a/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala b/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala
new file mode 100644
index 0000000..bfe6995
--- /dev/null
+++ b/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala
@@ -0,0 +1,57 @@
+package xyz.driver.core.file
+
+import java.io.File
+import java.nio.file.{Path, Paths}
+
+import xyz.driver.core.{Name, Revision}
+import xyz.driver.core.time.Time
+
+import scala.concurrent.{ExecutionContext, Future}
+import scalaz.{ListT, OptionT}
+
+class FileSystemStorage(executionContext: ExecutionContext) extends FileStorage {
+ implicit private val execution = executionContext
+
+ def upload(localSource: File, destination: Path): Future[Unit] = Future {
+ checkSafeFileName(destination) {
+ val destinationFile = destination.toFile
+
+ if (destinationFile.getParentFile.exists() || destinationFile.getParentFile.mkdirs()) {
+ if (localSource.renameTo(destinationFile)) ()
+ else {
+ throw new Exception(
+ s"Failed to move file from `${localSource.getCanonicalPath}` to `${destinationFile.getCanonicalPath}`")
+ }
+ } else {
+ throw new Exception(s"Failed to create parent directories for file `${destinationFile.getCanonicalPath}`")
+ }
+ }
+ }
+
+ def download(filePath: Path): OptionT[Future, File] =
+ OptionT.optionT(Future {
+ Option(new File(filePath.toString)).filter(file => file.exists() && file.isFile)
+ })
+
+ def delete(filePath: Path): Future[Unit] = Future {
+ val file = new File(filePath.toString)
+ if (file.delete()) ()
+ else {
+ throw new Exception(s"Failed to delete file $file" + (if (!file.exists()) ", file does not exist." else "."))
+ }
+ }
+
+ def list(path: Path): ListT[Future, FileLink] =
+ ListT.listT(Future {
+ val file = new File(path.toString)
+ if (file.isDirectory) {
+ file.listFiles().toList.filter(_.isFile).map { file =>
+ FileLink(Name[File](file.getName),
+ Paths.get(file.getPath),
+ Revision[File](file.hashCode.toString),
+ Time(file.lastModified()),
+ file.length())
+ }
+ } else List.empty[FileLink]
+ })
+}
diff --git a/src/main/scala/xyz/driver/core/file/S3Storage.scala b/src/main/scala/xyz/driver/core/file/S3Storage.scala
new file mode 100644
index 0000000..50bfe85
--- /dev/null
+++ b/src/main/scala/xyz/driver/core/file/S3Storage.scala
@@ -0,0 +1,64 @@
+package xyz.driver.core.file
+
+import java.io.File
+import java.nio.file.{Path, Paths}
+import java.util.UUID.randomUUID
+
+import com.amazonaws.services.s3.AmazonS3
+import com.amazonaws.services.s3.model.{Bucket, GetObjectRequest, ListObjectsV2Request}
+import xyz.driver.core.{Name, Revision}
+import xyz.driver.core.time.Time
+
+import scala.concurrent.{ExecutionContext, Future}
+import scalaz.{ListT, OptionT}
+
+class S3Storage(s3: AmazonS3, bucket: Name[Bucket], executionContext: ExecutionContext) extends FileStorage {
+ implicit private val execution = executionContext
+
+ def upload(localSource: File, destination: Path): Future[Unit] = Future {
+ checkSafeFileName(destination) {
+ val _ = s3.putObject(bucket.value, destination.toString, localSource).getETag
+ }
+ }
+
+ def download(filePath: Path): OptionT[Future, File] =
+ OptionT.optionT(Future {
+ val tempDir = System.getProperty("java.io.tmpdir")
+ val randomFolderName = randomUUID().toString
+ val tempDestinationFile = new File(Paths.get(tempDir, randomFolderName, filePath.toString).toString)
+
+ if (!tempDestinationFile.getParentFile.mkdirs()) {
+ throw new Exception(s"Failed to create temp directory to download file `$tempDestinationFile`")
+ } else {
+ Option(s3.getObject(new GetObjectRequest(bucket.value, filePath.toString), tempDestinationFile)).map { _ =>
+ tempDestinationFile
+ }
+ }
+ })
+
+ def delete(filePath: Path): Future[Unit] = Future {
+ s3.deleteObject(bucket.value, filePath.toString)
+ }
+
+ def list(path: Path): ListT[Future, FileLink] =
+ ListT.listT(Future {
+ import scala.collection.JavaConverters._
+ val req = new ListObjectsV2Request().withBucketName(bucket.value).withPrefix(path.toString).withMaxKeys(2)
+
+ def isInSubFolder(path: Path)(fileLink: FileLink) =
+ fileLink.location.toString.replace(path.toString + "/", "").contains("/")
+
+ Iterator.continually(s3.listObjectsV2(req)).takeWhile { result =>
+ req.setContinuationToken(result.getNextContinuationToken)
+ result.isTruncated
+ } flatMap { result =>
+ result.getObjectSummaries.asScala.toList.map { summary =>
+ FileLink(Name[File](summary.getKey),
+ Paths.get(path.toString + "/" + summary.getKey),
+ Revision[File](summary.getETag),
+ Time(summary.getLastModified.getTime),
+ summary.getSize)
+ } filterNot isInSubFolder(path)
+ } toList
+ })
+}
diff --git a/src/main/scala/xyz/driver/core/file/package.scala b/src/main/scala/xyz/driver/core/file/package.scala
new file mode 100644
index 0000000..8c06010
--- /dev/null
+++ b/src/main/scala/xyz/driver/core/file/package.scala
@@ -0,0 +1,51 @@
+package xyz.driver.core
+
+import java.io.File
+import java.nio.file.Path
+
+import xyz.driver.core.time.Time
+
+import scala.concurrent.Future
+import scalaz.{ListT, OptionT}
+
+package file {
+ final case class FileLink(
+ name: Name[File],
+ location: Path,
+ revision: Revision[File],
+ lastModificationDate: Time,
+ fileSize: Long
+ )
+
+ trait FileService {
+
+ def getFileLink(id: Name[File]): FileLink
+
+ def getFile(fileLink: FileLink): File
+ }
+
+ trait FileStorage {
+
+ def upload(localSource: File, destination: Path): Future[Unit]
+
+ def download(filePath: Path): OptionT[Future, File]
+
+ def delete(filePath: Path): Future[Unit]
+
+ def list(path: Path): ListT[Future, FileLink]
+
+ /** List of characters to avoid in S3 (I would say file names in general)
+ *
+ * @see http://stackoverflow.com/questions/7116450/what-are-valid-s3-key-names-that-can-be-accessed-via-the-s3-rest-api
+ */
+ private val illegalChars = "\\^`><{}][#%~|&@:,$=+?; "
+
+ protected def checkSafeFileName[T](filePath: Path)(f: => T): T = {
+ filePath.toString.find(c => illegalChars.contains(c)) match {
+ case Some(illegalCharacter) =>
+ throw new IllegalArgumentException(s"File name cannot contain character `$illegalCharacter`")
+ case None => f
+ }
+ }
+ }
+}