From 914b6f702f7f2e2eafc3c384387dc88df65fab9f Mon Sep 17 00:00:00 2001 From: vlad Date: Tue, 16 Aug 2016 21:08:08 -0700 Subject: File storage at S3 and File System --- src/main/scala/com/drivergrp/core/app.scala | 8 +- src/main/scala/com/drivergrp/core/file.scala | 124 ++++++++++++++++++++++++++- 2 files changed, 125 insertions(+), 7 deletions(-) (limited to 'src/main/scala/com') diff --git a/src/main/scala/com/drivergrp/core/app.scala b/src/main/scala/com/drivergrp/core/app.scala index a3ccd9e..4d1b775 100644 --- a/src/main/scala/com/drivergrp/core/app.scala +++ b/src/main/scala/com/drivergrp/core/app.scala @@ -68,8 +68,8 @@ object app { log.debug(s"Request is not allowed to $uri ($requestUuid)", is) complete( - HttpResponse(BadRequest, - entity = s"""{ "requestUuid": "$requestUuid", "message": "${is.getMessage}" }""")) + HttpResponse(BadRequest, + entity = s"""{ "requestUuid": "$requestUuid", "message": "${is.getMessage}" }""")) } case cm: ConcurrentModificationException => @@ -79,8 +79,8 @@ object app { log.debug(s"Concurrent modification of the resource $uri ($requestUuid)", cm) complete( - HttpResponse(Conflict, - entity = s"""{ "requestUuid": "$requestUuid", "message": "${cm.getMessage}" }""")) + HttpResponse(Conflict, + entity = s"""{ "requestUuid": "$requestUuid", "message": "${cm.getMessage}" }""")) } case t: Throwable => diff --git a/src/main/scala/com/drivergrp/core/file.scala b/src/main/scala/com/drivergrp/core/file.scala index c085be8..207570a 100644 --- a/src/main/scala/com/drivergrp/core/file.scala +++ b/src/main/scala/com/drivergrp/core/file.scala @@ -1,16 +1,24 @@ package com.drivergrp.core -import akka.http.scaladsl.model.Uri +import java.io.File +import java.nio.file.Paths +import java.util.UUID._ + +import com.amazonaws.services.s3.AmazonS3 +import com.amazonaws.services.s3.model.{Bucket, GetObjectRequest, ListObjectsV2Request} import com.drivergrp.core.time.Time +import scala.concurrent.{ExecutionContext, Future} +import scalaz.ListT + object file { - final case class File(id: Id[File]) + final case class FilePath(path: String) final case class FileLink( id: Id[File], name: Name[File], - location: Uri, + location: FilePath, additionDate: Time ) @@ -20,4 +28,114 @@ object file { def getFile(fileLink: FileLink): File } + + trait FileStorage { + + def upload(localSource: File, destination: FilePath): Future[Unit] + + def download(filePath: FilePath): Future[File] + + def delete(filePath: FilePath): Future[Unit] + + def list(path: FilePath): ListT[Future, Name[File]] + + /** List of characters to avoid in S3 (I would say file names in general) + * + * @see http://stackoverflow.com/questions/7116450/what-are-valid-s3-key-names-that-can-be-accessed-via-the-s3-rest-api + */ + private val illegalChars = "\\^`><{}][#%~|&@:,$=+?; " + + protected def checkSafeFileName[T](filePath: FilePath)(f: => T): T = { + filePath.path.find(c => illegalChars.contains(c)) match { + case Some(illegalCharacter) => + throw new IllegalArgumentException(s"File name cannot contain character `$illegalCharacter`") + case None => f + } + } + } + + class S3Storage(s3: AmazonS3, bucket: Name[Bucket], executionContext: ExecutionContext) extends FileStorage { + implicit private val execution = executionContext + + def upload(localSource: File, destination: FilePath): Future[Unit] = Future { + checkSafeFileName(destination) { + val _ = s3.putObject(bucket, destination.path, localSource).getETag + } + } + + def download(filePath: FilePath): Future[File] = Future { + val tempDir = System.getProperty("java.io.tmpdir") + val randomFolderName = randomUUID().toString + val tempDestinationFile = new File(Paths.get(tempDir, randomFolderName, filePath.path).toString) + + if (!tempDestinationFile.getParentFile.mkdirs()) { + throw new Exception(s"Failed to create temp directory to download file `$file`") + } else { + val _ = s3.getObject(new GetObjectRequest(bucket, filePath.path), tempDestinationFile) + tempDestinationFile + } + } + + def delete(filePath: FilePath): Future[Unit] = Future { + s3.deleteObject(bucket, filePath.path) + } + + def list(path: FilePath): ListT[Future, Name[File]] = + ListT.listT(Future { + import scala.collection.JavaConverters._ + val req = new ListObjectsV2Request().withBucketName(bucket).withPrefix(path.path).withMaxKeys(2) + + Iterator.continually(s3.listObjectsV2(req)).takeWhile { result => + req.setContinuationToken(result.getNextContinuationToken) + result.isTruncated + } flatMap { result => + result.getObjectSummaries.asScala + .map(_.getKey) + .toList + .filterNot(_.replace(path.path + "/", "").contains("/")) // filter out sub-folders or files in sub-folders + .map(item => Name[File](new File(item).getName)) + } toList + }) + } + + class FileSystemStorage(executionContext: ExecutionContext) extends FileStorage { + implicit private val execution = executionContext + + def upload(localSource: File, destination: FilePath): Future[Unit] = Future { + checkSafeFileName(destination) { + val destinationFile = Paths.get(destination.path).toFile + + if (destinationFile.getParentFile.exists() || destinationFile.getParentFile.mkdirs()) { + if (localSource.renameTo(destinationFile)) () + else { + throw new Exception( + s"Failed to move file from `${localSource.getCanonicalPath}` to `${destinationFile.getCanonicalPath}`") + } + } else { + throw new Exception(s"Failed to create parent directories for file `${destinationFile.getCanonicalPath}`") + } + } + } + + def download(filePath: FilePath): Future[File] = Future { + make(new File(filePath.path)) { file => + assert(file.exists() && file.isFile) + } + } + + def delete(filePath: FilePath): Future[Unit] = Future { + val file = new File(filePath.path) + if (file.delete()) () + else { + throw new Exception(s"Failed to delete file $file" + (if (!file.exists()) ", file does not exist." else ".")) + } + } + + def list(path: FilePath): ListT[Future, Name[File]] = + ListT.listT(Future { + val file = new File(path.path) + if (file.isDirectory) file.listFiles().filter(_.isFile).map(f => Name[File](f.getName)).toList + else List.empty[Name[File]] + }) + } } -- cgit v1.2.3