diff options
Diffstat (limited to 'src/main/scala/xyz/driver/pdsuicommon')
11 files changed, 354 insertions, 16 deletions
diff --git a/src/main/scala/xyz/driver/pdsuicommon/db/SlickPostgresQueryBuilder.scala b/src/main/scala/xyz/driver/pdsuicommon/db/SlickPostgresQueryBuilder.scala index f882441..a56ab9a 100644 --- a/src/main/scala/xyz/driver/pdsuicommon/db/SlickPostgresQueryBuilder.scala +++ b/src/main/scala/xyz/driver/pdsuicommon/db/SlickPostgresQueryBuilder.scala @@ -2,8 +2,7 @@ package xyz.driver.pdsuicommon.db import java.time.{LocalDateTime, ZoneOffset} -import slick.driver.JdbcProfile -import slick.jdbc.GetResult +import slick.jdbc.{JdbcProfile, GetResult} import xyz.driver.core.database.SlickDal import xyz.driver.pdsuicommon.logging._ diff --git a/src/main/scala/xyz/driver/pdsuicommon/db/SlickQueryBuilder.scala b/src/main/scala/xyz/driver/pdsuicommon/db/SlickQueryBuilder.scala index ab2757b..0daa84d 100644 --- a/src/main/scala/xyz/driver/pdsuicommon/db/SlickQueryBuilder.scala +++ b/src/main/scala/xyz/driver/pdsuicommon/db/SlickQueryBuilder.scala @@ -3,8 +3,7 @@ package xyz.driver.pdsuicommon.db import java.sql.PreparedStatement import java.time.LocalDateTime -import slick.driver.JdbcProfile -import slick.jdbc.{PositionedParameters, SQLActionBuilder, SetParameter} +import slick.jdbc.{JdbcProfile, PositionedParameters, SQLActionBuilder, SetParameter} import xyz.driver.pdsuicommon.db.Sorting.{Dimension, Sequential} import xyz.driver.pdsuicommon.db.SortingOrder.{Ascending, Descending} diff --git a/src/main/scala/xyz/driver/pdsuicommon/parsers/SearchFilterParser.scala b/src/main/scala/xyz/driver/pdsuicommon/parsers/SearchFilterParser.scala index e0adeb8..e46e11c 100644 --- a/src/main/scala/xyz/driver/pdsuicommon/parsers/SearchFilterParser.scala +++ b/src/main/scala/xyz/driver/pdsuicommon/parsers/SearchFilterParser.scala @@ -15,10 +15,7 @@ object SearchFilterParser { private object BinaryAtomFromTuple { def unapply(input: (SearchFilterExpr.Dimension, (String, Any))): Option[SearchFilterExpr.Atom.Binary] = { val (dimensionName, (strOperation, value)) = input - val updatedValue = value match { - case s: String => s.safeTrim - case a => a - } + val updatedValue = trimIfString(value) parseOperation(strOperation.toLowerCase).map { op => SearchFilterExpr.Atom.Binary(dimensionName, op, updatedValue.asInstanceOf[AnyRef]) @@ -30,15 +27,24 @@ object SearchFilterParser { // Compiler warning: unchecked since it is eliminated by erasure, if we user Seq[String] def unapply(input: (SearchFilterExpr.Dimension, (String, Seq[_]))): Option[SearchFilterExpr.Atom.NAry] = { val (dimensionName, (strOperation, xs)) = input + val updatedValues = xs.map(trimIfString) + if (strOperation.toLowerCase == "in") { - val values = xs.asInstanceOf[Seq[String]].map(_.safeTrim) - Some(SearchFilterExpr.Atom.NAry(dimensionName, SearchFilterNAryOperation.In, values)) + Some( + SearchFilterExpr.Atom + .NAry(dimensionName, SearchFilterNAryOperation.In, updatedValues.map(_.asInstanceOf[AnyRef]))) } else { None } } } + private def trimIfString(value: Any) = + value match { + case s: String => s.safeTrim + case a => a + } + private val operationsMapping = { import xyz.driver.pdsuicommon.db.SearchFilterBinaryOperation._ @@ -96,7 +102,7 @@ object SearchFilterParser { private val nAryValueParser: Parser[String] = P(CharPred(_ != ',').rep(min = 1).!) - private val longParser: Parser[Long] = P(CharIn('0' to '9').rep(1).!.map(_.toLong)) + private val longParser: Parser[Long] = P(CharIn('0' to '9').rep(min = 1).!.map(_.toLong)) private val binaryAtomParser: Parser[SearchFilterExpr.Atom.Binary] = P( dimensionParser ~ whitespaceParser ~ ( @@ -109,7 +115,8 @@ object SearchFilterParser { private val nAryAtomParser: Parser[SearchFilterExpr.Atom.NAry] = P( dimensionParser ~ whitespaceParser ~ ( - naryOperatorParser ~/ whitespaceParser ~/ nAryValueParser.!.rep(min = 1, sep = ",") + naryOperatorParser ~ whitespaceParser ~ + (longParser.rep(min = 1, sep = ",") | nAryValueParser.!.rep(min = 1, sep = ",")) ) ~ End ).map { case NAryAtomFromTuple(atom) => atom diff --git a/src/main/scala/xyz/driver/pdsuicommon/synchronization/db/SlickDataSource.scala b/src/main/scala/xyz/driver/pdsuicommon/synchronization/db/SlickDataSource.scala new file mode 100644 index 0000000..63514ec --- /dev/null +++ b/src/main/scala/xyz/driver/pdsuicommon/synchronization/db/SlickDataSource.scala @@ -0,0 +1,23 @@ +package xyz.driver.pdsuicommon.synchronization.db + +import slick.dbio.DBIO + +import scalaz.OptionT + +trait SlickDataSource[T] { + + val isDictionary: Boolean = false + + /** + * @return New entity + */ + def create(x: T): DBIO[T] + + /** + * @return Updated entity + */ + def update(x: T): OptionT[DBIO, T] + + def delete(x: T): OptionT[DBIO, Unit] + +} diff --git a/src/main/scala/xyz/driver/pdsuicommon/synchronization/db/SlickDbAction.scala b/src/main/scala/xyz/driver/pdsuicommon/synchronization/db/SlickDbAction.scala new file mode 100644 index 0000000..57cc3d4 --- /dev/null +++ b/src/main/scala/xyz/driver/pdsuicommon/synchronization/db/SlickDbAction.scala @@ -0,0 +1,70 @@ +package xyz.driver.pdsuicommon.synchronization.db + +import slick.dbio.DBIO +import xyz.driver.pdsuicommon.logging._ +import xyz.driver.pdsuicommon.synchronization.utils.{FakeIdGen, FakeIdMap} + +import scala.concurrent.ExecutionContext +import scalaz.Monad + +trait SlickDbAction[+T] { + def entity: T +} + +object SlickDbAction { + + final case class Create[T](entity: T) extends SlickDbAction[T] + final case class Update[T](entity: T) extends SlickDbAction[T] + final case class Delete[T](entity: T) extends SlickDbAction[T] + + // Use it only inside of a transaction! + def unsafeRun[T](actions: List[SlickDbAction[T]], dataSource: SlickDataSource[T])( + implicit core: FakeIdGen[T], + executionContext: ExecutionContext, + dbioMonad: Monad[DBIO]): DBIO[FakeIdMap[T]] = { + unsafeRun(DBIO.successful(FakeIdMap.empty))(actions, dataSource) + } + + // Use it only inside of a transaction! + def unsafeRun[T](initial: DBIO[FakeIdMap[T]])(actions: List[SlickDbAction[T]], dataSource: SlickDataSource[T])( + implicit core: FakeIdGen[T], + executionContext: ExecutionContext, + dbioMonad: Monad[DBIO]): DBIO[FakeIdMap[T]] = { + // TODO Squash Updates and Delete to one operation, when bugs in repositories will be fixed + actions.foldLeft(initial) { + case (previousActions, Create(x)) => + for { + r <- previousActions + newArm <- dataSource.create(x) + } yield { + r + (core(newArm) -> newArm) + } + + case (previousActions, Update(x)) => + for { + r <- previousActions + updatedArm <- dataSource.update(x).getOrElse(x) + } yield { + r - core(updatedArm) + (core(updatedArm) -> updatedArm) + } + + case (previousActions, Delete(_)) if dataSource.isDictionary => + previousActions // We don't delete entities from dictionaries + + case (previousActions, Delete(x)) => + for { + r <- previousActions + _ <- dataSource.delete(x).run + } yield { + r - core(x) + } + } + } + + implicit def toPhiString[T](input: SlickDbAction[T])(implicit inner: T => PhiString): PhiString = input match { + case Create(x) => phi"Create($x)" + case Update(x) => phi"Update($x)" + case Delete(x) => phi"Delete($x)" + } + +} diff --git a/src/main/scala/xyz/driver/pdsuicommon/synchronization/db/SlickDbDiff.scala b/src/main/scala/xyz/driver/pdsuicommon/synchronization/db/SlickDbDiff.scala new file mode 100644 index 0000000..c226659 --- /dev/null +++ b/src/main/scala/xyz/driver/pdsuicommon/synchronization/db/SlickDbDiff.scala @@ -0,0 +1,52 @@ +package xyz.driver.pdsuicommon.synchronization.db + +import xyz.driver.pdsuicommon.synchronization.domain.FakeId +import xyz.driver.pdsuicommon.synchronization.utils.{FakeIdGen, Refiner} + +import scala.annotation.tailrec +import scala.collection.breakOut +import scala.collection.immutable.SortedSet + +object SlickDbDiff { + + /** + * Calculates DB-actions to synchronize origEntities with draftEntities. + */ + def calc[DraftT, OrigT](origEntities: Iterable[OrigT], draftEntities: Iterable[DraftT])( + implicit draftFakeIdGen: FakeIdGen[DraftT], + origFakeIdGen: FakeIdGen[OrigT], + refiner: Refiner[DraftT, OrigT]): List[SlickDbAction[OrigT]] = { + val origMap: Map[FakeId, OrigT] = origEntities.map(x => origFakeIdGen(x) -> x)(breakOut) + val uniqueDraftEntities = SortedSet.newBuilder[DraftT](Ordering.by[DraftT, FakeId](draftFakeIdGen)) + uniqueDraftEntities ++= draftEntities + + loop(origMap, uniqueDraftEntities.result(), List.empty) + } + + @tailrec private def loop[DraftT, OrigT](origEntitiesMap: Map[FakeId, OrigT], + draftEntities: Iterable[DraftT], + actions: List[SlickDbAction[OrigT]])( + implicit draftFakeIdGen: FakeIdGen[DraftT], + refiner: Refiner[DraftT, OrigT]): List[SlickDbAction[OrigT]] = { + draftEntities.headOption match { + case None => + // The rest original entities are not a part of draft, so we will delete them + val toDelete: List[SlickDbAction[OrigT]] = origEntitiesMap.values.map(x => SlickDbAction.Delete(x))(breakOut) + actions ++ toDelete + + case Some(currRaw) => + val rawCore = draftFakeIdGen.getFor(currRaw) + val action: Option[SlickDbAction[OrigT]] = origEntitiesMap.get(rawCore) match { + // It is a new entity, because it doesn't exist among originals + case None => Some(SlickDbAction.Create(refiner.refine(currRaw))) + case Some(orig) => + val draft = refiner.refresh(orig, currRaw) + if (draft == orig) None + else Some(SlickDbAction.Update(draft)) + } + + loop(origEntitiesMap - rawCore, draftEntities.tail, action.map(_ :: actions).getOrElse(actions)) + } + } + +} diff --git a/src/main/scala/xyz/driver/pdsuicommon/synchronization/domain/FakeId.scala b/src/main/scala/xyz/driver/pdsuicommon/synchronization/domain/FakeId.scala new file mode 100644 index 0000000..38e442b --- /dev/null +++ b/src/main/scala/xyz/driver/pdsuicommon/synchronization/domain/FakeId.scala @@ -0,0 +1,14 @@ +package xyz.driver.pdsuicommon.synchronization.domain + +/* + It is like an Id for entities those haven't an Id, but should be unique. + For example, + RawArm has the name, the kind and the intervention fields. + It has not an Id, but should be identified by the name field. + So, the name field is a fake id for RawArm. + */ +final case class FakeId(value: String) + +object FakeId { + implicit val ordering: Ordering[FakeId] = Ordering.by(_.value) +} diff --git a/src/main/scala/xyz/driver/pdsuicommon/synchronization/utils/FakeIdGen.scala b/src/main/scala/xyz/driver/pdsuicommon/synchronization/utils/FakeIdGen.scala new file mode 100644 index 0000000..196aab1 --- /dev/null +++ b/src/main/scala/xyz/driver/pdsuicommon/synchronization/utils/FakeIdGen.scala @@ -0,0 +1,26 @@ +package xyz.driver.pdsuicommon.synchronization.utils + +import xyz.driver.pdsuicommon.synchronization.domain.FakeId + +/** + * Used to generate a fake id from an entity. + * A fake id is used in comparison between entities with different types, + * for example, RawTrial and Trial. + * + * @see FakeId + */ +trait FakeIdGen[-T] extends (T => FakeId) { + + def getFor(x: T): FakeId + + override def apply(x: T): FakeId = getFor(x) + +} + +object FakeIdGen { + + def create[T](f: T => FakeId) = new FakeIdGen[T] { + override def getFor(x: T): FakeId = f(x) + } + +} diff --git a/src/main/scala/xyz/driver/pdsuicommon/synchronization/utils/Refiner.scala b/src/main/scala/xyz/driver/pdsuicommon/synchronization/utils/Refiner.scala new file mode 100644 index 0000000..768b889 --- /dev/null +++ b/src/main/scala/xyz/driver/pdsuicommon/synchronization/utils/Refiner.scala @@ -0,0 +1,12 @@ +package xyz.driver.pdsuicommon.synchronization.utils + +/** + * Allows to extract a data from the From entity to convert/update in to the To entity. + */ +trait Refiner[-From, To] { + + def refine(raw: From): To + + def refresh(orig: To, update: From): To + +} diff --git a/src/main/scala/xyz/driver/pdsuicommon/synchronization/utils/package.scala b/src/main/scala/xyz/driver/pdsuicommon/synchronization/utils/package.scala new file mode 100644 index 0000000..1b30158 --- /dev/null +++ b/src/main/scala/xyz/driver/pdsuicommon/synchronization/utils/package.scala @@ -0,0 +1,64 @@ +package xyz.driver.pdsuicommon.synchronization + +import java.net.URL +import java.nio.ByteBuffer +import java.util.UUID + +import xyz.driver.pdsuicommon.domain.{LongId, UuidId} +import xyz.driver.pdsuicommon.http.HttpFetcher +import xyz.driver.pdsuicommon.json.JsonSerializer +import xyz.driver.pdsuicommon.synchronization.domain.FakeId + +import scala.collection.breakOut +import scala.concurrent.{ExecutionContext, Future} +import scala.io.Codec + +package object utils { + + type FakeIdMap[T] = Map[FakeId, T] + + object FakeIdMap { + + def empty[T]: FakeIdMap[T] = Map.empty + + def create[T](xs: Seq[T])(implicit fakeIdExtractor: FakeIdGen[T]): FakeIdMap[T] = { + xs.map({ x => + fakeIdExtractor.getFor(x) -> x + })(breakOut) + } + + } + + /** + * Requests domain objects from the repository using + * ids of fetched dictionary entities + * + * @param getList repository access function + * @param xs sequence of entity objects + * @param id function that extracts id from the entity + * @tparam Id Type of Id (for example [[LongId]], [[UuidId]]) + * @tparam K Type parameter for Id + * @tparam D Domain object type name + * @tparam E Dictionary entity object type name + */ + def domainFromEntities[K, D, E, Id[_]](getList: Set[Id[K]] => Seq[D], xs: Seq[E])(id: E => Id[K]): Seq[D] = { + getList(xs.map(x => id(x)).toSet) + } + + /** Version of [[domainFromEntities]] for LongId */ + def domainFromEntitiesLong[K, D, E](getList: Set[LongId[K]] => Seq[D], xs: Seq[E])(id: E => Long): Seq[D] = { + domainFromEntities(getList, xs)(e => LongId(id(e))) + } + + /** Version of [[domainFromEntities]] for UuidId */ + def domainFromEntitiesUUID[K, D, E](getList: Set[UuidId[K]] => Seq[D], xs: Seq[E])(id: E => UUID): Seq[D] = { + domainFromEntities(getList, xs)(e => UuidId(id(e))) + } + + def fetch[T](httpFetcher: HttpFetcher, url: URL)(implicit m: Manifest[T], ec: ExecutionContext): Future[T] = { + httpFetcher(url).map { rawContent => + val content = Codec.UTF8.decoder.decode(ByteBuffer.wrap(rawContent)).toString + JsonSerializer.deserialize[T](content) + } + } +} diff --git a/src/main/scala/xyz/driver/pdsuicommon/utils/CustomSwaggerJsonFormats.scala b/src/main/scala/xyz/driver/pdsuicommon/utils/CustomSwaggerJsonFormats.scala index 6c87858..83c4e6f 100644 --- a/src/main/scala/xyz/driver/pdsuicommon/utils/CustomSwaggerJsonFormats.scala +++ b/src/main/scala/xyz/driver/pdsuicommon/utils/CustomSwaggerJsonFormats.scala @@ -4,7 +4,7 @@ import java.time.{LocalDate, LocalDateTime} import io.swagger.models.properties.Property import spray.json.JsValue -import xyz.driver.pdsuicommon.domain.{LongId, StringId, UuidId} +import xyz.driver.pdsuicommon.domain.{LongId, StringId, TextJson, UuidId} import xyz.driver.pdsuidomain.entities._ import xyz.driver.pdsuidomain.formats.json.sprayformats.arm._ import xyz.driver.pdsuidomain.formats.json.sprayformats.criterion._ @@ -15,25 +15,29 @@ import xyz.driver.pdsuidomain.formats.json.sprayformats.trial._ import xyz.driver.pdsuidomain.formats.json.sprayformats.trialhistory._ import xyz.driver.pdsuidomain.formats.json.sprayformats.trialissue._ import xyz.driver.core.swagger.CustomSwaggerJsonConverter._ +import xyz.driver.pdsuicommon.concurrent.BridgeUploadQueue import xyz.driver.pdsuidomain.services.CriterionService.RichCriterion +import xyz.driver.pdsuidomain.services.ExtractedDataService.RichExtractedData + +import scala.collection.immutable object CustomSwaggerJsonFormats { - val customCommonProperties = Map[Class[_], Property]( + val customCommonProperties = immutable.Map[Class[_], Property]( classOf[LocalDateTime] -> stringProperty(example = Some("2010-12-31'T'18:59:59Z")), classOf[LocalDate] -> stringProperty(example = Some("2010-12-31")), classOf[UuidId[_]] -> stringProperty(example = Some("370b0450-35cb-4aab-ba74-0145be75add5")), classOf[StringId[_]] -> stringProperty(), classOf[LongId[_]] -> stringProperty() ) - val customTrialCurationProperties = Map[Class[_], Property]( + val customTrialCurationProperties = immutable.Map[Class[_], Property]( classOf[Trial.Status] -> stringProperty(), classOf[Trial.Condition] -> stringProperty(), classOf[TrialHistory.Action] -> stringProperty(), classOf[TrialHistory.State] -> stringProperty() ) ++ customCommonProperties - val customTrialCurationObjectsExamples = Map[Class[_], JsValue]( + val customTrialCurationObjectsExamples = immutable.Map[Class[_], JsValue]( classOf[Trial] -> trialWriter.write(xyz.driver.pdsuidomain.fakes.entities.trialcuration.nextTrial()), classOf[Arm] -> armFormat.write(xyz.driver.pdsuidomain.fakes.entities.trialcuration.nextArm()), classOf[TrialHistory] -> trialHistoryFormat.write( @@ -52,4 +56,72 @@ object CustomSwaggerJsonFormats { xyz.driver.pdsuidomain.fakes.entities.trialcuration.nextStudyDesign()) ) + // records-processing-service + object Rep { + import xyz.driver.pdsuidomain.fakes.entities.rep + import xyz.driver.pdsuidomain.formats.json.sprayformats.document + import xyz.driver.pdsuidomain.formats.json.sprayformats.documentissue + import xyz.driver.pdsuidomain.formats.json.sprayformats.documenthistory + import xyz.driver.pdsuidomain.formats.json.sprayformats.record + import xyz.driver.pdsuidomain.formats.json.sprayformats.recordissue + import xyz.driver.pdsuidomain.formats.json.sprayformats.recordhistory + import xyz.driver.pdsuidomain.formats.json.sprayformats.bridgeuploadqueue + import xyz.driver.pdsuidomain.formats.json.sprayformats.extracteddata + + val customRepObjectsExamples = immutable.Map[Class[_], JsValue]( + classOf[Document] -> + document.documentFormat.write(rep.DocumentGen.nextDocument()), + classOf[Document.Meta] -> + document.documentMetaFormat.write(rep.DocumentGen.nextDocumentMeta()), + classOf[TextJson[Document.Meta]] -> + document.fullDocumentMetaFormat.write(rep.DocumentGen.nextDocumentMetaJson()), + classOf[Document.RequiredType] -> + document.requiredTypeFormat.write(rep.DocumentGen.nextDocumentRequiredType()), + classOf[Document.Status] -> + document.documentStatusFormat.write(rep.DocumentGen.nextDocumentStatus()), + classOf[DocumentIssue] -> + documentissue.documentIssueFormat.write(rep.DocumentGen.nextDocumentIssue()), + classOf[DocumentHistory] -> + documenthistory.documentHistoryFormat.write(rep.DocumentGen.nextDocumentHistory()), + classOf[DocumentHistory.Action] -> + documenthistory.documentActionFormat.write(rep.DocumentGen.nextDocumentHistoryAction()), + classOf[DocumentHistory.State] -> + documenthistory.documentStateFormat.write(rep.DocumentGen.nextDocumentHistoryState()), + classOf[ProviderType] -> + record.providerTypeFormat.write(rep.MedicalRecordGen.nextProviderType()), + classOf[TextJson[List[MedicalRecord.Meta]]] -> + record.recordMetaFormat.write(rep.MedicalRecordGen.nextMedicalRecordMetasJson()), + classOf[MedicalRecord] -> + record.recordFormat.write(rep.MedicalRecordGen.nextMedicalRecord()), + classOf[MedicalRecord.Meta] -> + record.recordMetaTypeFormat.write(rep.MedicalRecordGen.nextMedicalRecordMeta()), + classOf[MedicalRecord.Status] -> + record.recordStatusFormat.write(rep.MedicalRecordGen.nextMedicalRecordStatus()), + classOf[MedicalRecordIssue] -> + recordissue.recordIssueFormat.write(rep.MedicalRecordGen.nextMedicalRecordIssue()), + classOf[MedicalRecordHistory] -> + recordhistory.recordHistoryFormat.write(rep.MedicalRecordGen.nextMedicalRecordHistory()), + classOf[MedicalRecordHistory.Action] -> + recordhistory.recordActionFormat.write(rep.MedicalRecordGen.nextMedicalRecordHistoryAction()), + classOf[MedicalRecordHistory.State] -> + recordhistory.recordStateFormat.write(rep.MedicalRecordGen.nextMedicalRecordHistoryState()), + classOf[BridgeUploadQueue.Item] -> + bridgeuploadqueue.queueUploadItemFormat.write(rep.BridgeUploadQueueGen.nextBridgeUploadQueueItem()), + classOf[ExtractedData.Meta] -> + extracteddata.extractedDataMetaFormat.write(rep.ExtractedDataGen.nextExtractedDataMeta()), + classOf[ExtractedData.Meta.Evidence] -> + extracteddata.metaEvidenceFormat.write(rep.ExtractedDataGen.nextExtractedDataMetaEvidence()), + classOf[ExtractedData.Meta.Keyword] -> + extracteddata.metaKeywordFormat.write(rep.ExtractedDataGen.nextExtractedDataMetaKeyword()), + classOf[ExtractedData.Meta.TextLayerPosition] -> + extracteddata.metaTextLayerPositionFormat.write(rep.ExtractedDataGen.nextExtractedDataMetaTextLayerPosition()), + classOf[TextJson[ExtractedData.Meta]] -> + extracteddata.fullExtractedDataMetaFormat.write(rep.ExtractedDataGen.nextExtractedDataMetaJson()), + classOf[RichExtractedData] -> + extracteddata.extractedDataFormat.write(rep.ExtractedDataGen.nextRichExtractedData()), + classOf[ExtractedDataLabel] -> + extracteddata.extractedDataLabelWriter.write(rep.ExtractedDataGen.nextExtractedDataLabel()) + ) + } + } |