From 4f953ddecb4831720fab7a0994738e7529ac0a99 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Thu, 18 Aug 2016 12:13:00 -0400 Subject: [PATCH 1/6] filter jes metadata events by specific events --- engine/src/main/scala/cromwell/engine/backend/jes/Run.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala b/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala index 5175572d8..86eb87bbf 100644 --- a/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala +++ b/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala @@ -1,8 +1,8 @@ package cromwell.engine.backend.jes import com.google.api.client.util.ArrayMap +import com.google.api.services.genomics.model.{CancelOperationRequest, LoggingOptions, RunPipelineArgs, RunPipelineRequest, ServiceAccount, _} import com.google.api.services.genomics.{Genomics, model} -import com.google.api.services.genomics.model.{CancelOperationRequest, LoggingOptions, Pipeline, RunPipelineArgs, RunPipelineRequest, ServiceAccount, _} import com.typesafe.config.ConfigFactory import cromwell.core.WorkflowId import cromwell.engine.backend.BackendCallJobDescriptor @@ -119,6 +119,9 @@ object Run { } toSeq } else Seq.empty + val acceptableEvents = Set("start", "pulling-image", "localizing-files", "running-docker", "delocalizing-files", "ok", "fail", "start-shutdown") + val filteredEventsList: Seq[EventStartTime] = { eventsList filter { i => acceptableEvents.contains(i.name) } } + // The final event is only used as the book-end for the final pairing (see below) so the name is never actually used... // ... which is rather a pity actually - it's a jolly good name. val finaleEvents = eventIfExists("endTime", op, "cromwell poll interval") ++ Seq( From 1afe80931dee2a24af7eca41b70183ec73e4e987 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Thu, 18 Aug 2016 13:42:35 -0400 Subject: [PATCH 2/6] use the filtered list --- engine/src/main/scala/cromwell/engine/backend/jes/Run.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala b/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala index 86eb87bbf..9c1ba3c61 100644 --- a/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala +++ b/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala @@ -128,7 +128,7 @@ object Run { EventStartTime("The Queen flying around with a jet-pack, with Winston Churchill cheering and waving a huge Union Jack in the background", DateTime.now)) // Join the Seqs together, pair up consecutive elements then make events with start and end times. - ((starterEvents ++ eventsList ++ finaleEvents).sliding(2) toSeq) map { case Seq(a, b) => ExecutionEventEntry(a.name, a.timestamp, b.timestamp) } + ((starterEvents ++ filteredEventsList ++ finaleEvents).sliding(2) toSeq) map { case Seq(a, b) => ExecutionEventEntry(a.name, a.timestamp, b.timestamp) } } private def eventIfExists(name: String, op: Operation, eventName: String): Seq[EventStartTime] = { From 002de00ddde5b2d233251e4e136d2db75bb616c5 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Thu, 18 Aug 2016 13:50:03 -0400 Subject: [PATCH 3/6] pr changes --- engine/src/main/scala/cromwell/engine/backend/jes/Run.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala b/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala index 9c1ba3c61..d84d946eb 100644 --- a/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala +++ b/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala @@ -26,6 +26,7 @@ object Run { lazy val MaximumPollingInterval = Duration(ConfigFactory.load.getConfig("backend").getConfig("jes").getInt("maximumPollingInterval"), "seconds") val InitialPollingInterval = 5 seconds val PollingBackoffFactor = 1.1 + val acceptableEvents = Set("start", "pulling-image", "localizing-files", "running-docker", "delocalizing-files", "ok", "fail", "start-shutdown") def apply(runIdForResumption: Option[String], jesJobDescriptor: JesJobDescriptor, @@ -119,8 +120,7 @@ object Run { } toSeq } else Seq.empty - val acceptableEvents = Set("start", "pulling-image", "localizing-files", "running-docker", "delocalizing-files", "ok", "fail", "start-shutdown") - val filteredEventsList: Seq[EventStartTime] = { eventsList filter { i => acceptableEvents.contains(i.name) } } + val filteredEventsList: Seq[EventStartTime] = eventsList filter { i => acceptableEvents.contains(i.name) } // The final event is only used as the book-end for the final pairing (see below) so the name is never actually used... // ... which is rather a pity actually - it's a jolly good name. From e964af4884c1971eefdcea5617637dcb679dbae3 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Thu, 18 Aug 2016 15:22:15 -0400 Subject: [PATCH 4/6] capitals --- engine/src/main/scala/cromwell/engine/backend/jes/Run.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala b/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala index d84d946eb..27c0245e3 100644 --- a/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala +++ b/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala @@ -26,7 +26,7 @@ object Run { lazy val MaximumPollingInterval = Duration(ConfigFactory.load.getConfig("backend").getConfig("jes").getInt("maximumPollingInterval"), "seconds") val InitialPollingInterval = 5 seconds val PollingBackoffFactor = 1.1 - val acceptableEvents = Set("start", "pulling-image", "localizing-files", "running-docker", "delocalizing-files", "ok", "fail", "start-shutdown") + val AcceptableEvents = Set("start", "pulling-image", "localizing-files", "running-docker", "delocalizing-files", "ok", "fail", "start-shutdown") def apply(runIdForResumption: Option[String], jesJobDescriptor: JesJobDescriptor, From 32c2478003a5b945a7778a8cb8e39152012e85c6 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Thu, 18 Aug 2016 15:30:26 -0400 Subject: [PATCH 5/6] fix RunSpec --- .../cromwell/engine/backend/jes/RunSpec.scala | 35 +++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/engine/src/test/scala/cromwell/engine/backend/jes/RunSpec.scala b/engine/src/test/scala/cromwell/engine/backend/jes/RunSpec.scala index 7576deac1..9b77d4c60 100644 --- a/engine/src/test/scala/cromwell/engine/backend/jes/RunSpec.scala +++ b/engine/src/test/scala/cromwell/engine/backend/jes/RunSpec.scala @@ -14,11 +14,11 @@ class RunSpec extends FlatSpec with Matchers { val op: Operation = new Operation() val event1: ArrayMap[String, String] = ArrayMap.create(2) - event1.add("description", "blah") + event1.add("description", "pulling-image") event1.add("startTime", "2015-12-05T00:00:01+00:00") val event2: ArrayMap[String, String] = ArrayMap.create(2) - event2.add("description", "blah2") + event2.add("description", "start") event2.add("startTime", "2015-12-05T00:01:00+00:00") val events = new util.ArrayList(Seq(event1, event2).asJava) @@ -30,14 +30,41 @@ class RunSpec extends FlatSpec with Matchers { "events" -> events ) - op.setMetadata(metadata.asJava) Run.getEventList(op) should have size 5 - Run.getEventList(op) filter { x => x.description == "blah" } foreach { x => + Run.getEventList(op) filter { x => x.description == "pulling-image" } foreach { x => x.startTime.getMillis should be (new DateTime("2015-12-05T00:00:01.000Z").getMillis) x.endTime.getMillis should be (new DateTime("2015-12-05T00:01:00.000Z").getMillis) } + } + + "JES Run" should "not parse unwated events from Operation metadata" in { + val op: Operation = new Operation() + + val event1: ArrayMap[String, String] = ArrayMap.create(2) + event1.add("description", "blah") + event1.add("startTime", "2015-12-05T00:00:01+00:00") + + val event2: ArrayMap[String, String] = ArrayMap.create(2) + event2.add("description", "blah2") + event2.add("startTime", "2015-12-05T00:01:00+00:00") + + val events = new util.ArrayList(Seq(event1, event2).asJava) + + val metadata: Map[String, AnyRef] = Map( + "createTime" -> "2015-12-05T00:00:00+00:00", + "startTime" -> "2015-12-05T00:00:01+00:00", + "endTime" -> "2015-12-05T11:00:00+00:00", + "events" -> events + ) + op.setMetadata(metadata.asJava) + + Run.getEventList(op) should have size 3 + Run.getEventList(op) filter { x => x.description == "blah" } foreach { x => + x.startTime.getMillis should be (None) + x.endTime.getMillis should be (None) + } } } From 3419b45e51f6491d11842d944ad47e20c55e5ca0 Mon Sep 17 00:00:00 2001 From: Ruchi Munshi Date: Thu, 18 Aug 2016 15:43:24 -0400 Subject: [PATCH 6/6] rename properly --- engine/src/main/scala/cromwell/engine/backend/jes/Run.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala b/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala index 27c0245e3..5663f71ec 100644 --- a/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala +++ b/engine/src/main/scala/cromwell/engine/backend/jes/Run.scala @@ -120,7 +120,7 @@ object Run { } toSeq } else Seq.empty - val filteredEventsList: Seq[EventStartTime] = eventsList filter { i => acceptableEvents.contains(i.name) } + val filteredEventsList: Seq[EventStartTime] = eventsList filter { i => AcceptableEvents.contains(i.name) } // The final event is only used as the book-end for the final pairing (see below) so the name is never actually used... // ... which is rather a pity actually - it's a jolly good name.