diff options
author | Jakob Odersky <jakob@driver.xyz> | 2018-06-29 15:26:09 -0700 |
---|---|---|
committer | Jakob Odersky <jakob@driver.xyz> | 2018-06-29 15:26:09 -0700 |
commit | 901b02274fdfc08030443aac2f1760fc479b3816 (patch) | |
tree | 5d5f6d6d58fc9caa22eb836ccf70936a5b45ab81 /src/main/scala/xyz/driver/core/file | |
parent | 981cc63b94c6df5fc8f4d2f6ebafd1a6f27d7c4e (diff) | |
download | driver-core-901b02274fdfc08030443aac2f1760fc479b3816.tar.gz driver-core-901b02274fdfc08030443aac2f1760fc479b3816.tar.bz2 driver-core-901b02274fdfc08030443aac2f1760fc479b3816.zip |
Add build support for ScalaJS
Diffstat (limited to 'src/main/scala/xyz/driver/core/file')
4 files changed, 0 insertions, 366 deletions
diff --git a/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala b/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala deleted file mode 100644 index ce26fe4..0000000 --- a/src/main/scala/xyz/driver/core/file/FileSystemStorage.scala +++ /dev/null @@ -1,76 +0,0 @@ -package xyz.driver.core.file - -import akka.NotUsed -import akka.stream.scaladsl.{FileIO, Source} -import akka.util.ByteString -import java.io.File -import java.nio.file.{Files, Path, Paths} - -import xyz.driver.core.{Name, Revision} -import xyz.driver.core.time.Time - -import scala.concurrent.{ExecutionContext, Future} -import scalaz.{ListT, OptionT} - -@deprecated("Consider using xyz.driver.core.storage.FileSystemBlobStorage instead", "driver-core 1.8.14") -class FileSystemStorage(executionContext: ExecutionContext) extends FileStorage { - implicit private val execution = executionContext - - override def upload(localSource: File, destination: Path): Future[Unit] = Future { - checkSafeFileName(destination) { - val destinationFile = destination.toFile - - if (destinationFile.getParentFile.exists() || destinationFile.getParentFile.mkdirs()) { - if (localSource.renameTo(destinationFile)) () - else { - throw new Exception( - s"Failed to move file from `${localSource.getCanonicalPath}` to `${destinationFile.getCanonicalPath}`") - } - } else { - throw new Exception(s"Failed to create parent directories for file `${destinationFile.getCanonicalPath}`") - } - } - } - - override def download(filePath: Path): OptionT[Future, File] = - OptionT.optionT(Future { - Option(new File(filePath.toString)).filter(file => file.exists() && file.isFile) - }) - - override def stream(filePath: Path): OptionT[Future, Source[ByteString, NotUsed]] = - OptionT.optionT(Future { - if (Files.exists(filePath)) { - Some(FileIO.fromPath(filePath).mapMaterializedValue(_ => NotUsed)) - } else { - None - } - }) - - override def delete(filePath: Path): Future[Unit] = Future { - val file = new File(filePath.toString) - if (file.delete()) () - else { - throw new Exception(s"Failed to delete file $file" + (if (!file.exists()) ", file does not exist." else ".")) - } - } - - override def list(path: Path): ListT[Future, FileLink] = - ListT.listT(Future { - val file = new File(path.toString) - if (file.isDirectory) { - file.listFiles().toList.filter(_.isFile).map { file => - FileLink( - Name[File](file.getName), - Paths.get(file.getPath), - Revision[File](file.hashCode.toString), - Time(file.lastModified()), - file.length()) - } - } else List.empty[FileLink] - }) - - override def exists(path: Path): Future[Boolean] = Future { - Files.exists(path) - } - -} diff --git a/src/main/scala/xyz/driver/core/file/GcsStorage.scala b/src/main/scala/xyz/driver/core/file/GcsStorage.scala deleted file mode 100644 index 5c94645..0000000 --- a/src/main/scala/xyz/driver/core/file/GcsStorage.scala +++ /dev/null @@ -1,135 +0,0 @@ -package xyz.driver.core.file - -import akka.NotUsed -import akka.stream.scaladsl.Source -import akka.util.ByteString -import com.google.cloud.ReadChannel -import java.io.{BufferedOutputStream, File, FileInputStream, FileOutputStream} -import java.net.URL -import java.nio.ByteBuffer -import java.nio.file.{Path, Paths} -import java.util.concurrent.TimeUnit - -import com.google.cloud.storage.Storage.BlobListOption -import com.google.cloud.storage.{Option => _, _} -import xyz.driver.core.time.Time -import xyz.driver.core.{Name, Revision, generators} - -import scala.collection.JavaConverters._ -import scala.concurrent.duration.Duration -import scala.concurrent.{ExecutionContext, Future} -import scalaz.{ListT, OptionT} - -@deprecated("Consider using xyz.driver.core.storage.GcsBlobStorage instead", "driver-core 1.8.14") -class GcsStorage( - storageClient: Storage, - bucketName: Name[Bucket], - executionContext: ExecutionContext, - chunkSize: Int = 4096) - extends SignedFileStorage { - implicit private val execution: ExecutionContext = executionContext - - override def upload(localSource: File, destination: Path): Future[Unit] = Future { - checkSafeFileName(destination) { - val blobId = BlobId.of(bucketName.value, destination.toString) - def acl = Bucket.BlobWriteOption.predefinedAcl(Storage.PredefinedAcl.PUBLIC_READ) - - storageClient.get(bucketName.value).create(blobId.getName, new FileInputStream(localSource), acl) - } - } - - override def download(filePath: Path): OptionT[Future, File] = { - OptionT.optionT(Future { - Option(storageClient.get(bucketName.value, filePath.toString)).filterNot(_.getSize == 0).map { - blob => - val tempDir = System.getProperty("java.io.tmpdir") - val randomFolderName = generators.nextUuid().toString - val tempDestinationFile = new File(Paths.get(tempDir, randomFolderName, filePath.toString).toString) - - if (!tempDestinationFile.getParentFile.mkdirs()) { - throw new Exception(s"Failed to create temp directory to download file `$tempDestinationFile`") - } else { - val target = new BufferedOutputStream(new FileOutputStream(tempDestinationFile)) - try target.write(blob.getContent()) - finally target.close() - tempDestinationFile - } - } - }) - } - - override def stream(filePath: Path): OptionT[Future, Source[ByteString, NotUsed]] = - OptionT.optionT(Future { - def readChunk(rc: ReadChannel): Option[ByteString] = { - val buffer = ByteBuffer.allocate(chunkSize) - val length = rc.read(buffer) - if (length > 0) { - buffer.flip() - Some(ByteString.fromByteBuffer(buffer)) - } else { - None - } - } - - Option(storageClient.get(bucketName.value, filePath.toString)).map { blob => - Source.unfoldResource[ByteString, ReadChannel]( - create = () => blob.reader(), - read = channel => readChunk(channel), - close = channel => channel.close() - ) - } - }) - - override def delete(filePath: Path): Future[Unit] = Future { - storageClient.delete(BlobId.of(bucketName.value, filePath.toString)) - } - - override def list(directoryPath: Path): ListT[Future, FileLink] = - ListT.listT(Future { - val directory = s"$directoryPath/" - val page = storageClient.list( - bucketName.value, - BlobListOption.currentDirectory(), - BlobListOption.prefix(directory) - ) - - page - .iterateAll() - .asScala - .filter(_.getName != directory) - .map(blobToFileLink(directoryPath, _)) - .toList - }) - - protected def blobToFileLink(path: Path, blob: Blob): FileLink = { - def nullError(property: String) = throw new IllegalStateException(s"Blob $blob at $path does not have $property") - val name = Option(blob.getName).getOrElse(nullError("a name")) - val generation = Option(blob.getGeneration).getOrElse(nullError("a generation")) - val updateTime = Option(blob.getUpdateTime).getOrElse(nullError("an update time")) - val size = Option(blob.getSize).getOrElse(nullError("a size")) - - FileLink( - Name(name.split('/').last), - Paths.get(name), - Revision(generation.toString), - Time(updateTime), - size - ) - } - - override def exists(path: Path): Future[Boolean] = Future { - val blob = Option( - storageClient.get( - bucketName.value, - path.toString - )) - blob.isDefined - } - - override def signedFileUrl(filePath: Path, duration: Duration): OptionT[Future, URL] = - OptionT.optionT(Future { - Option(storageClient.get(bucketName.value, filePath.toString)).filterNot(_.getSize == 0).map { blob => - blob.signUrl(duration.toSeconds, TimeUnit.SECONDS) - } - }) -} diff --git a/src/main/scala/xyz/driver/core/file/S3Storage.scala b/src/main/scala/xyz/driver/core/file/S3Storage.scala deleted file mode 100644 index 5158d4d..0000000 --- a/src/main/scala/xyz/driver/core/file/S3Storage.scala +++ /dev/null @@ -1,87 +0,0 @@ -package xyz.driver.core.file - -import akka.NotUsed -import akka.stream.scaladsl.{Source, StreamConverters} -import akka.util.ByteString -import java.io.File -import java.nio.file.{Path, Paths} -import java.util.UUID.randomUUID - -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.{Bucket, GetObjectRequest, ListObjectsV2Request} -import xyz.driver.core.{Name, Revision} -import xyz.driver.core.time.Time - -import scala.concurrent.{ExecutionContext, Future} -import scalaz.{ListT, OptionT} - -@deprecated( - "Blob storage functionality has been reimplemented in xyz.driver.core.storage.BlobStorage. " + - "It has not been ported to S3 storage. Please raise an issue if this required for your use-case.", - "driver-core 1.8.14" -) -class S3Storage(s3: AmazonS3, bucket: Name[Bucket], executionContext: ExecutionContext, chunkSize: Int = 4096) - extends FileStorage { - implicit private val execution = executionContext - - override def upload(localSource: File, destination: Path): Future[Unit] = Future { - checkSafeFileName(destination) { - val _ = s3.putObject(bucket.value, destination.toString, localSource).getETag - } - } - - override def download(filePath: Path): OptionT[Future, File] = - OptionT.optionT(Future { - val tempDir = System.getProperty("java.io.tmpdir") - val randomFolderName = randomUUID().toString - val tempDestinationFile = new File(Paths.get(tempDir, randomFolderName, filePath.toString).toString) - - if (!tempDestinationFile.getParentFile.mkdirs()) { - throw new Exception(s"Failed to create temp directory to download file `$tempDestinationFile`") - } else { - Option(s3.getObject(new GetObjectRequest(bucket.value, filePath.toString), tempDestinationFile)).map { _ => - tempDestinationFile - } - } - }) - - override def stream(filePath: Path): OptionT[Future, Source[ByteString, NotUsed]] = - OptionT.optionT(Future { - Option(s3.getObject(new GetObjectRequest(bucket.value, filePath.toString))).map { elem => - StreamConverters.fromInputStream(() => elem.getObjectContent(), chunkSize).mapMaterializedValue(_ => NotUsed) - } - }) - - override def delete(filePath: Path): Future[Unit] = Future { - s3.deleteObject(bucket.value, filePath.toString) - } - - override def list(path: Path): ListT[Future, FileLink] = - ListT.listT(Future { - import scala.collection.JavaConverters._ - val req = new ListObjectsV2Request().withBucketName(bucket.value).withPrefix(path.toString).withMaxKeys(2) - - def isInSubFolder(path: Path)(fileLink: FileLink) = - fileLink.location.toString.replace(path.toString + "/", "").contains("/") - - Iterator.continually(s3.listObjectsV2(req)).takeWhile { result => - req.setContinuationToken(result.getNextContinuationToken) - result.isTruncated - } flatMap { result => - result.getObjectSummaries.asScala.toList.map { summary => - FileLink( - Name[File](summary.getKey), - Paths.get(path.toString + "/" + summary.getKey), - Revision[File](summary.getETag), - Time(summary.getLastModified.getTime), - summary.getSize - ) - } filterNot isInSubFolder(path) - } toList - }) - - override def exists(path: Path): Future[Boolean] = Future { - s3.doesObjectExist(bucket.value, path.toString) - } - -} diff --git a/src/main/scala/xyz/driver/core/file/package.scala b/src/main/scala/xyz/driver/core/file/package.scala deleted file mode 100644 index 58955e5..0000000 --- a/src/main/scala/xyz/driver/core/file/package.scala +++ /dev/null @@ -1,68 +0,0 @@ -package xyz.driver.core - -import java.io.File -import java.nio.file.Path - -import xyz.driver.core.time.Time - -import scala.concurrent.Future -import scalaz.{ListT, OptionT} - -package file { - - import akka.NotUsed - import akka.stream.scaladsl.Source - import akka.util.ByteString - import java.net.URL - - import scala.concurrent.duration.Duration - - final case class FileLink( - name: Name[File], - location: Path, - revision: Revision[File], - lastModificationDate: Time, - fileSize: Long - ) - - trait FileService { - - def getFileLink(id: Name[File]): FileLink - - def getFile(fileLink: FileLink): File - } - - trait FileStorage { - - def upload(localSource: File, destination: Path): Future[Unit] - - def download(filePath: Path): OptionT[Future, File] - - def stream(filePath: Path): OptionT[Future, Source[ByteString, NotUsed]] - - def delete(filePath: Path): Future[Unit] - - /** List contents of a directory */ - def list(directoryPath: Path): ListT[Future, FileLink] - - def exists(path: Path): Future[Boolean] - - /** List of characters to avoid in S3 (I would say file names in general) - * - * @see http://stackoverflow.com/questions/7116450/what-are-valid-s3-key-names-that-can-be-accessed-via-the-s3-rest-api - */ - private val illegalChars = "\\^`><{}][#%~|&@:,$=+?; " - - protected def checkSafeFileName[T](filePath: Path)(f: => T): T = { - filePath.toString.find(c => illegalChars.contains(c)) match { - case Some(illegalCharacter) => - throw new IllegalArgumentException(s"File name cannot contain character `$illegalCharacter`") - case None => f - } - } - } - - trait SignedFileStorage extends FileStorage { - def signedFileUrl(filePath: Path, duration: Duration): OptionT[Future, URL] - } -} |