diff --git a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json index 34d52f8f052..f1a899d2123 100644 --- a/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json +++ b/ansible/files/whisks_design_document_for_entities_db_v2.1.0.json @@ -21,6 +21,9 @@ "triggers": { "map": "function (doc) {\n var PATHSEP = \"/\";\n var isTrigger = function (doc) { return (doc.exec === undefined && doc.binding === undefined && doc.parameters !== undefined) };\n if (isTrigger(doc)) try {\n var ns = doc.namespace.split(PATHSEP);\n var root = ns[0];\n var value = {\n namespace: doc.namespace,\n name: doc.name,\n version: doc.version,\n publish: doc.publish,\n annotations: doc.annotations,\n updated: doc.updated\n };\n emit([doc.namespace, doc.updated], value);\n if (root !== doc.namespace) {\n emit([root, doc.updated], value);\n }\n } catch (e) {}\n}", "reduce": "_count" + }, + "action-versions": { + "map": "function (doc) {\n var isAction = function (doc) { return (doc.exec !== undefined) };\n if (isAction(doc)) try {\n var value = {\n _id: doc.namespace + \"/\" + doc.name + \"/default\",\n namespace: doc.namespace,\n name: doc.name,\n docId: doc._id,\n version: doc.version\n };\n emit([doc.namespace + \"/\" + doc.name], value);\n } catch (e) {}\n}" } } -} \ No newline at end of file +} diff --git a/common/scala/src/main/resources/application.conf b/common/scala/src/main/resources/application.conf index 140d4c4b023..986ff1c8934 100644 --- a/common/scala/src/main/resources/application.conf +++ b/common/scala/src/main/resources/application.conf @@ -109,6 +109,7 @@ kamon { whisk { shared-packages-execute-only = false + action-maximum-versions = 10 metrics { # Enable/disable Prometheus support. If enabled then metrics would be exposed at `/metrics` endpoint # If Prometheus is enabled then please review `kamon.metric.tick-interval` (set to 1 sec by default above). diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/WarmUp.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/WarmUp.scala index cd2e205c561..77d59f0ade4 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/WarmUp.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/WarmUp.scala @@ -41,6 +41,7 @@ object WarmUp { revision = DocRevision.empty, user = warmUpActionIdentity, activationId = new ActivationIdGenerator {}.make(), + DocId(warmUpAction.asString), rootControllerIndex = controller, blocking = false, content = None, @@ -54,6 +55,7 @@ object WarmUp { val metadata = WhiskActionMetaData( warmUpAction.path, warmUpAction.name, + DocId(warmUpAction.asString), CodeExecMetaDataAsString(manifest, false, entryPoint = None)) ContainerCreationMessage( TransactionId.warmUp, diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala index 6a16886fcf4..d28182400e8 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/WhiskConfig.scala @@ -294,6 +294,7 @@ object ConfigKeys { val whiskConfig = "whisk.config" val sharedPackageExecuteOnly = s"whisk.shared-packages-execute-only" + val actionVersionLimit = "whisk.action-maximum-versions" val swaggerUi = "whisk.swagger-ui" /* DEPRECATED: disableStoreResult is deprecated for storeBlockingResultLevel */ diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/connector/Message.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/connector/Message.scala index 9de194798cd..995edc72c30 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/connector/Message.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/connector/Message.scala @@ -55,6 +55,7 @@ case class ActivationMessage(override val transid: TransactionId, revision: DocRevision, user: Identity, activationId: ActivationId, + actionId: DocId, rootControllerIndex: ControllerInstanceId, blocking: Boolean, content: Option[JsValue], @@ -193,7 +194,7 @@ object ActivationMessage extends DefaultJsonProtocol { def parse(msg: String) = Try(serdes.read(msg.parseJson)) private implicit val fqnSerdes = FullyQualifiedEntityName.serdes - implicit val serdes = jsonFormat12(ActivationMessage.apply) + implicit val serdes = jsonFormat13(ActivationMessage.apply) } object CombinedCompletionAndResultMessage extends DefaultJsonProtocol { diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala index f61aacc6a2d..ff63468285c 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/DocumentHandler.scala @@ -96,13 +96,17 @@ abstract class SimpleHandler extends DocumentHandler { provider: DocumentProvider)(implicit transid: TransactionId, ec: ExecutionContext): Future[Seq[JsObject]] = { //Query result from CouchDB have below object structure with actual result in `value` key //So transform the result to confirm to that structure - val viewResult = JsObject( - "id" -> js.fields("_id"), - "key" -> createKey(ddoc, view, startKey, js), - "value" -> computeView(ddoc, view, js)) + val value = computeView(ddoc, view, js) + val viewResult = JsObject("id" -> js.fields("_id"), "key" -> createKey(ddoc, view, startKey, js), "value" -> value) - val result = if (includeDocs) JsObject(viewResult.fields + ("doc" -> js)) else viewResult - Future.successful(Seq(result)) + if (includeDocs) value.fields.get("_id") match { + case Some(JsString(id)) if id != js.fields("_id") => + provider.get(DocId(id)).map { doc => + Seq(JsObject(viewResult.fields + ("doc" -> doc.getOrElse(js)))) + } + case _ => + Future.successful(Seq(JsObject(viewResult.fields + ("doc" -> js)))) + } else Future.successful(Seq(viewResult)) } /** @@ -115,6 +119,28 @@ abstract class SimpleHandler extends DocumentHandler { * Key is an array which matches the view query key */ protected def createKey(ddoc: String, view: String, startKey: List[Any], js: JsObject): JsArray + + /** + * Finds and transforms annotation with matching key. + * + * @param js js object having annotations array + * @param key annotation key + * @param vtr transformer function to map annotation value + * @param default default value to use if no matching annotation found + * @return annotation value matching given key + */ + protected[database] def annotationValue[T](js: JsObject, key: String, vtr: JsValue => T, default: T): T = { + js.fields.get("annotations") match { + case Some(JsArray(e)) => + e.view + .map(_.asJsObject.getFields("key", "value")) + .collectFirst { + case Seq(JsString(`key`), v: JsValue) => vtr(v) //match annotation with given key + } + .getOrElse(default) + case _ => default + } + } } object ActivationHandler extends SimpleHandler { @@ -177,35 +203,15 @@ object ActivationHandler extends SimpleHandler { }, name) } - /** - * Finds and transforms annotation with matching key. - * - * @param js js object having annotations array - * @param key annotation key - * @param vtr transformer function to map annotation value - * @param default default value to use if no matching annotation found - * @return annotation value matching given key - */ - protected[database] def annotationValue[T](js: JsObject, key: String, vtr: JsValue => T, default: T): T = { - js.fields.get("annotations") match { - case Some(JsArray(e)) => - e.view - .map(_.asJsObject.getFields("key", "value")) - .collectFirst { - case Seq(JsString(`key`), v: JsValue) => vtr(v) //match annotation with given key - } - .getOrElse(default) - case _ => default - } - } - private def dropNull(fields: JsField*) = JsObject(fields.filter(_._2 != JsNull): _*) } object WhisksHandler extends SimpleHandler { val ROOT_NS = "rootns" + val FULL_NAME = "fullname" private val commonFields = Set("namespace", "name", "version", "publish", "annotations", "updated") private val actionFields = commonFields ++ Set("limits", "exec.binary") + private val actionVersionFields = commonFields ++ Set("_id", "docId") private val packageFields = commonFields ++ Set("binding") private val packagePublicFields = commonFields private val ruleFields = commonFields @@ -213,6 +219,7 @@ object WhisksHandler extends SimpleHandler { protected val supportedTables = Set( "whisks.v2.1.0/actions", + "whisks.v2.1.0/action-versions", "whisks.v2.1.0/packages", "whisks.v2.1.0/packages-public", "whisks.v2.1.0/rules", @@ -223,13 +230,20 @@ object WhisksHandler extends SimpleHandler { case Some(JsString(namespace)) => val ns = namespace.split(PATHSEP) val rootNS = if (ns.length > 1) ns(0) else namespace - JsObject((ROOT_NS, JsString(rootNS))) + js.fields.get("name") match { + case Some(JsString(name)) => + val fullName = s"$namespace$PATHSEP$name" + JsObject((ROOT_NS, JsString(rootNS)), (FULL_NAME, JsString(fullName))) + case _ => + JsObject((ROOT_NS, JsString(rootNS))) + } case _ => JsObject.empty } } override def fieldsRequiredForView(ddoc: String, view: String): Set[String] = view match { case "actions" => actionFields + case "action-versions" => actionVersionFields case "packages" => packageFields case "packages-public" => packagePublicFields case "rules" => ruleFields @@ -239,6 +253,7 @@ object WhisksHandler extends SimpleHandler { def computeView(ddoc: String, view: String, js: JsObject): JsObject = view match { case "actions" => computeActionView(js) + case "action-versions" => computeActionVersionsView(js) case "packages" => computePackageView(js) case "packages-public" => computePublicPackageView(js) case "rules" => computeRulesView(js) @@ -256,6 +271,7 @@ object WhisksHandler extends SimpleHandler { def getEntityTypeForDesignDoc(ddoc: String, view: String): String = view match { case "actions" => "action" + case "action-versions" => "action" case "rules" => "rule" case "triggers" => "trigger" case "packages" | "packages-public" => "package" @@ -288,6 +304,12 @@ object WhisksHandler extends SimpleHandler { val exec_binary = JsHelpers.getFieldPath(js, "exec", "binary") JsObject(base + ("exec" -> JsObject("binary" -> exec_binary.getOrElse(JsFalse)))) } + + private def computeActionVersionsView(js: JsObject): JsObject = { + val base = js.fields.filterKeys(actionVersionFields).toMap + val defaultId = js.fields("namespace") + "/" + js.fields("name") + "/default" + JsObject(base + ("_id" -> JsString(defaultId), "docId" -> js.fields("_id"))) + } } object SubjectHandler extends DocumentHandler { diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/RemoteCacheInvalidation.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/RemoteCacheInvalidation.scala index c5b5e019b3a..b081dbfe07c 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/RemoteCacheInvalidation.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/RemoteCacheInvalidation.scala @@ -32,13 +32,16 @@ import org.apache.openwhisk.core.{ConfigKeys, WhiskConfig} import org.apache.openwhisk.core.connector.Message import org.apache.openwhisk.core.connector.MessageFeed import org.apache.openwhisk.core.connector.MessagingProvider -import org.apache.openwhisk.core.entity.CacheKey -import org.apache.openwhisk.core.entity.ControllerInstanceId -import org.apache.openwhisk.core.entity.WhiskAction -import org.apache.openwhisk.core.entity.WhiskActionMetaData -import org.apache.openwhisk.core.entity.WhiskPackage -import org.apache.openwhisk.core.entity.WhiskRule -import org.apache.openwhisk.core.entity.WhiskTrigger +import org.apache.openwhisk.core.entity.{ + CacheKey, + ControllerInstanceId, + WhiskAction, + WhiskActionMetaData, + WhiskActionVersionList, + WhiskPackage, + WhiskRule, + WhiskTrigger +} import org.apache.openwhisk.spi.SpiLoader import pureconfig._ @@ -92,6 +95,7 @@ class RemoteCacheInvalidation(config: WhiskConfig, component: String, instance: WhiskPackage.removeId(msg.key) WhiskRule.removeId(msg.key) WhiskTrigger.removeId(msg.key) + WhiskActionVersionList.removeId(msg.key) } } case Failure(t) => logging.error(this, s"failed processing message: $raw with $t") diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBViewMapper.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBViewMapper.scala index 8e1e102f495..3bceb2ba452 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBViewMapper.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/cosmosdb/CosmosDBViewMapper.scala @@ -26,6 +26,7 @@ import kamon.metric.MeasurementUnit import org.apache.openwhisk.common.{LogMarkerToken, TransactionId, WhiskInstants} import org.apache.openwhisk.core.database.ActivationHandler.NS_PATH import org.apache.openwhisk.core.database.WhisksHandler.ROOT_NS +import org.apache.openwhisk.core.database.WhisksHandler.FULL_NAME import org.apache.openwhisk.core.database.cosmosdb.CosmosDBConstants.{alias, computed, deleted} import org.apache.openwhisk.core.database.{ ActivationHandler, @@ -143,6 +144,7 @@ private[cosmosdb] abstract class SimpleMapper extends CosmosDBViewMapper { private[cosmosdb] object WhisksViewMapper extends SimpleMapper { private val NS = "namespace" private val ROOT_NS_C = s"$computed.$ROOT_NS" + private val FULL_NAME_C = s"$computed.$FULL_NAME" private val TYPE = "entityType" private val UPDATED = "updated" private val PUBLISH = "publish" @@ -169,7 +171,8 @@ private[cosmosdb] object WhisksViewMapper extends SimpleMapper { viewConditions(ddoc, view).map(q => (s"${q._1} AND", q._2)).getOrElse((NOTHING, Nil)) val params = ("@entityType", entityType) :: ("@namespace", namespace) :: vcParams - val baseCondition = s"$vc r.$TYPE = @entityType AND (r.$NS = @namespace OR r.$ROOT_NS_C = @namespace)" + val baseCondition = + s"$vc r.$TYPE = @entityType AND (r.$NS = @namespace OR r.$ROOT_NS_C = @namespace OR r.$FULL_NAME_C = @namespace)" (startKey, endKey) match { case (_ :: Nil, _ :: `TOP` :: Nil) => @@ -194,7 +197,8 @@ private[cosmosdb] object WhisksViewMapper extends SimpleMapper { } override protected def orderByField(ddoc: String, view: String): String = view match { - case "actions" | "rules" | "triggers" | "packages" | "packages-public" if ddoc.startsWith("whisks") => + case "actions" | "action-versions" | "rules" | "triggers" | "packages" | "packages-public" + if ddoc.startsWith("whisks") => s"r.$UPDATED" case _ => throw UnsupportedView(s"$ddoc/$view") } diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryViewMapper.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryViewMapper.scala index 4898e723ede..ab0a328ff4b 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryViewMapper.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/database/memory/MemoryViewMapper.scala @@ -123,6 +123,7 @@ private object ActivationViewMapper extends MemoryViewMapper { private object WhisksViewMapper extends MemoryViewMapper { private val NS = "namespace" private val ROOT_NS = WhisksHandler.ROOT_NS + private val FULL_NAME = WhisksHandler.FULL_NAME private val TYPE = "entityType" private val UPDATED = "updated" private val PUBLISH = "publish" @@ -140,20 +141,23 @@ private object WhisksViewMapper extends MemoryViewMapper { val matchTypeAndView = equal(d, TYPE, entityType) && matchViewConditions(ddoc, view, d) val matchNS = equal(d, NS, startKey.head.asInstanceOf[String]) val matchRootNS = equal(c, ROOT_NS, startKey.head.asInstanceOf[String]) + val matchFullName = equal(c, FULL_NAME, startKey.head.asInstanceOf[String]) //Here ddocs for actions, rules and triggers use //namespace and namespace/packageName as first key val filterResult = (startKey, endKey) match { case (ns :: Nil, _ :: `TOP` :: Nil) => - (matchTypeAndView && matchNS) || (matchTypeAndView && matchRootNS) + (matchTypeAndView && matchNS) || (matchTypeAndView && matchRootNS) || (matchTypeAndView && matchFullName) case (ns :: (since: Number) :: Nil, _ :: `TOP` :: `TOP` :: Nil) => (matchTypeAndView && matchNS && gte(d, UPDATED, since)) || - (matchTypeAndView && matchRootNS && gte(d, UPDATED, since)) + (matchTypeAndView && matchRootNS && gte(d, UPDATED, since)) || + (matchTypeAndView && matchFullName && gte(d, UPDATED, since)) case (ns :: (since: Number) :: Nil, _ :: (upto: Number) :: `TOP` :: Nil) => (matchTypeAndView && matchNS && gte(d, UPDATED, since) && lte(d, UPDATED, upto)) || - (matchTypeAndView && matchRootNS && gte(d, UPDATED, since) && lte(d, UPDATED, upto)) + (matchTypeAndView && matchRootNS && gte(d, UPDATED, since) && lte(d, UPDATED, upto)) || + (matchTypeAndView && matchFullName && gte(d, UPDATED, since) && lte(d, UPDATED, upto)) case _ => throw UnsupportedQueryKeys(s"$ddoc/$view -> ($startKey, $endKey)") } @@ -177,7 +181,8 @@ private object WhisksViewMapper extends MemoryViewMapper { override def sort(ddoc: String, view: String, descending: Boolean, s: Seq[JsObject]): Seq[JsObject] = { view match { - case "actions" | "rules" | "triggers" | "packages" | "packages-public" if ddoc.startsWith("whisks") => + case "actions" | "action-versions" | "rules" | "triggers" | "packages" | "packages-public" + if ddoc.startsWith("whisks") => numericSort(s, descending, UPDATED) case _ => throw UnsupportedView(s"$ddoc/$view") } diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/SemVer.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/SemVer.scala index d9c5efd3034..334ae0305b6 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/SemVer.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/SemVer.scala @@ -21,6 +21,8 @@ import spray.json.deserializationError import spray.json.JsString import spray.json.JsValue import spray.json.RootJsonFormat + +import scala.annotation.tailrec import scala.util.Try /** @@ -34,7 +36,7 @@ import scala.util.Try * * @param (major, minor, patch) for the semantic version */ -protected[core] class SemVer private (private val version: (Int, Int, Int)) extends AnyVal { +protected[core] class SemVer private (private val version: (Int, Int, Int)) extends AnyVal with Ordered[SemVer] { protected[core] def major = version._1 protected[core] def minor = version._2 @@ -46,6 +48,24 @@ protected[core] class SemVer private (private val version: (Int, Int, Int)) exte protected[entity] def toJson = JsString(toString) override def toString = s"$major.$minor.$patch" + + protected[core] def versionList = Seq(major, minor, patch) + + override def compare(that: SemVer): Int = { + compareVersion(that) + } + + @tailrec + private def compareVersion(that: SemVer, depth: Int = 0): Int = { + require(depth >= 0 && depth <= 2, "depth exceed the limit of 0 to 2") + val result = versionList(depth) - that.versionList(depth) + if (result != 0) + result + else if (depth == 2) + 0 + else + compareVersion(that, depth + 1) + } } protected[core] object SemVer { @@ -78,7 +98,7 @@ protected[core] object SemVer { * @return SemVer instance * @thrown IllegalArgumentException if string is not a valid semantic version */ - protected[entity] def apply(str: String): SemVer = { + protected[core] def apply(str: String): SemVer = { try { val parts = if (str != null && str.nonEmpty) str.split('.') else Array[String]() val major = if (parts.size >= 1) parts(0).toInt else 0 diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala index e06057367c0..aba82be4a0f 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskAction.scala @@ -28,8 +28,19 @@ import scala.concurrent.Future import scala.util.{Failure, Success, Try} import spray.json._ import spray.json.DefaultJsonProtocol._ -import org.apache.openwhisk.common.TransactionId -import org.apache.openwhisk.core.database.{ArtifactStore, CacheChangeNotification, DocumentFactory, NoDocumentException} + +import org.apache.openwhisk.common.{Logging, TransactionId} +import org.apache.openwhisk.core.database.{ + ArtifactStore, + CacheChangeNotification, + DocumentFactory, + EvictionPolicy, + MultipleReadersSingleWriterCache, + NoDocumentException, + StaleParameter, + WriteTime +} + import org.apache.openwhisk.core.entity.Attachments._ import org.apache.openwhisk.core.entity.types.EntityStore @@ -103,8 +114,10 @@ abstract class WhiskActionLike(override val name: EntityName) extends WhiskEntit "annotations" -> annotations.toJson) } -abstract class WhiskActionLikeMetaData(override val name: EntityName) extends WhiskActionLike(name) { +abstract class WhiskActionLikeMetaData(override val name: EntityName, val docId: DocId) extends WhiskActionLike(name) { override def exec: ExecMetaDataBase + + override def docid = docId } /** @@ -140,6 +153,8 @@ case class WhiskAction(namespace: EntityPath, require(exec != null, "exec undefined") require(limits != null, "limits undefined") + override def docid = DocId(fullyQualifiedName(true).asString) + /** * Merges parameters (usually from package) with existing action parameters. * Existing parameters supersede those in p. @@ -170,7 +185,7 @@ case class WhiskAction(namespace: EntityPath, def toExecutableWhiskAction: Option[ExecutableWhiskAction] = exec match { case codeExec: CodeExec[_] => Some( - ExecutableWhiskAction(namespace, name, codeExec, parameters, limits, version, publish, annotations) + ExecutableWhiskAction(namespace, name, docid, codeExec, parameters, limits, version, publish, annotations) .revision[ExecutableWhiskAction](rev)) case _ => None } @@ -195,6 +210,7 @@ case class WhiskAction(namespace: EntityPath, @throws[IllegalArgumentException] case class WhiskActionMetaData(namespace: EntityPath, override val name: EntityName, + override val docId: DocId, exec: ExecMetaDataBase, parameters: Parameters = Parameters(), limits: ActionLimits = ActionLimits(), @@ -203,7 +219,7 @@ case class WhiskActionMetaData(namespace: EntityPath, annotations: Parameters = Parameters(), override val updated: Instant = WhiskEntity.currentMillis(), binding: Option[EntityPath] = None) - extends WhiskActionLikeMetaData(name) { + extends WhiskActionLikeMetaData(name, docId) { require(exec != null, "exec undefined") require(limits != null, "limits undefined") @@ -235,6 +251,7 @@ case class WhiskActionMetaData(namespace: EntityPath, ExecutableWhiskActionMetaData( namespace, name, + docId, execMetaData, parameters, limits, @@ -273,6 +290,7 @@ case class WhiskActionMetaData(namespace: EntityPath, @throws[IllegalArgumentException] case class ExecutableWhiskAction(namespace: EntityPath, override val name: EntityName, + docId: DocId, exec: CodeExec[_], parameters: Parameters = Parameters(), limits: ActionLimits = ActionLimits(), @@ -319,6 +337,7 @@ case class ExecutableWhiskAction(namespace: EntityPath, @throws[IllegalArgumentException] case class ExecutableWhiskActionMetaData(namespace: EntityPath, override val name: EntityName, + override val docId: DocId, exec: ExecMetaData, parameters: Parameters = Parameters(), limits: ActionLimits = ActionLimits(), @@ -326,13 +345,13 @@ case class ExecutableWhiskActionMetaData(namespace: EntityPath, publish: Boolean = false, annotations: Parameters = Parameters(), binding: Option[EntityPath] = None) - extends WhiskActionLikeMetaData(name) { + extends WhiskActionLikeMetaData(name, docId) { require(exec != null, "exec undefined") require(limits != null, "limits undefined") def toWhiskAction = - WhiskActionMetaData(namespace, name, exec, parameters, limits, version, publish, annotations, updated) + WhiskActionMetaData(namespace, name, docId, exec, parameters, limits, version, publish, annotations, updated) .revision[WhiskActionMetaData](rev) /** @@ -343,6 +362,123 @@ case class ExecutableWhiskActionMetaData(namespace: EntityPath, } +case class WhiskActionVersionList(namespace: EntityPath, + name: EntityName, + versionMappings: Map[SemVer, DocId], + defaultVersion: Option[SemVer]) { + def matchedDocId(version: Option[SemVer]): Option[DocId] = { + (version, defaultVersion) match { + case (Some(ver), _) => + versionMappings.get(ver) + case (None, Some(default)) => + versionMappings.get(default) + case (None, None) if versionMappings.nonEmpty => + Some(versionMappings.maxBy(_._1)._2) + case _ => + None + } + } +} + +object WhiskActionVersionList extends MultipleReadersSingleWriterCache[WhiskActionVersionList, DocInfo] { + override val evictionPolicy: EvictionPolicy = WriteTime + val collectionName = "action-versions" + lazy val viewName = WhiskQueries.entitiesView(collection = collectionName).name + implicit val serdes = jsonFormat(WhiskActionVersionList.apply, "namespace", "name", "versions", "defaultVersion") + + def cacheKey(action: FullyQualifiedEntityName): CacheKey = { + CacheKey(action.fullPath.asString) + } + + def get(action: FullyQualifiedEntityName, datastore: EntityStore, fromCache: Boolean = true)( + implicit transId: TransactionId): Future[WhiskActionVersionList] = { + implicit val logger: Logging = datastore.logging + implicit val ec = datastore.executionContext + + val startKey = List(action.fullPath.asString) + val endKey = List(action.fullPath.asString, WhiskQueries.TOP) + cacheLookup( + cacheKey(action), + datastore + .query( + viewName, + startKey = startKey, + endKey = endKey, + skip = 0, + limit = 0, + includeDocs = true, + descending = false, + reduce = false, + stale = StaleParameter.No) + .map { result => + val values = result.map { row => + row.fields("value").asJsObject() + } + val versions = values.map { value => + val docId = value.fields.get("docId").map(_.compactPrint) + val version = Try { value.fields.get("version").map(_.convertTo[SemVer]) } getOrElse None + (version, docId) match { + case (Some(ver), Some(id)) => + Some((ver, DocId(id))) + case _ => + None + } + } + + val defaultVersion = if (result.nonEmpty) { + result.head.fields.get("doc") match { + case Some(value) => Try { value.asJsObject.fields.get("default").map(_.convertTo[SemVer]) } getOrElse None + case None => None + } + } else None + WhiskActionVersionList(action.path, action.name, versions.filter(_.nonEmpty).map(_.get).toMap, defaultVersion) + }, + fromCache) + } + + def getMatchedDocId(action: FullyQualifiedEntityName, version: Option[SemVer], datastore: EntityStore)( + implicit transId: TransactionId, + ec: ExecutionContext): Future[Option[DocId]] = { + get(action, datastore).map { res => + res.matchedDocId(version) + } + } + + // delete cache + def deleteCache(action: FullyQualifiedEntityName)(implicit transId: TransactionId, + ec: ExecutionContext, + logger: Logging, + notifier: Option[CacheChangeNotification]) = { + cacheInvalidate(cacheKey(action), Future.successful(())) + } +} + +object WhiskActionDefaultVersion extends DocumentFactory[WhiskActionDefaultVersion] { + import WhiskActivation.instantSerdes + implicit val serdes = jsonFormat(WhiskActionDefaultVersion.apply, "namespace", "name", "default", "updated") +} + +case class WhiskActionDefaultVersion(namespace: EntityPath, + override val name: EntityName, + default: Option[SemVer] = None, + override val updated: Instant = WhiskEntity.currentMillis()) + extends WhiskEntity(name, "action-default-version") { + + /** + * The representation as JSON, e.g. for REST calls. Does not include id/rev. + */ + override def toJson: JsObject = WhiskActionDefaultVersion.serdes.write(this).asJsObject + + /** + * Gets unique document identifier for the document. + */ + override def docid: DocId = new DocId(namespace + EntityPath.PATHSEP + name + "/default") + + override val version: SemVer = SemVer() + override val publish: Boolean = true + override val annotations: Parameters = Parameters() +} + object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[WhiskAction] with DefaultJsonProtocol { import WhiskActivation.instantSerdes @@ -539,14 +675,17 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[ * If it's the actual package, use its name directly as the package path name. * While traversing the package bindings, merge the parameters. */ - def resolveActionAndMergeParameters(entityStore: EntityStore, fullyQualifiedName: FullyQualifiedEntityName)( - implicit ec: ExecutionContext, - transid: TransactionId): Future[WhiskAction] = { + def resolveActionAndMergeParameters( + entityStore: EntityStore, + fullyQualifiedName: FullyQualifiedEntityName, + version: Option[SemVer] = None)(implicit ec: ExecutionContext, transid: TransactionId): Future[WhiskAction] = { // first check that there is a package to be resolved val entityPath = fullyQualifiedName.path if (entityPath.defaultPackage) { // this is the default package, nothing to resolve - WhiskAction.get(entityStore, fullyQualifiedName.toDocId) + WhiskActionVersionList.getMatchedDocId(fullyQualifiedName, version, entityStore).flatMap { docId => + WhiskAction.get(entityStore, docId.getOrElse(fullyQualifiedName.toDocId)) + } } else { // there is a package to be resolved val pkgDocid = fullyQualifiedName.path.toDocId @@ -555,8 +694,11 @@ object WhiskAction extends DocumentFactory[WhiskAction] with WhiskEntityQueries[ wp flatMap { resolvedPkg => // fully resolved name for the action val fqnAction = resolvedPkg.fullyQualifiedName(withVersion = false).add(actionName) + val action = WhiskActionVersionList.getMatchedDocId(fqnAction, version, entityStore).flatMap { docId => + WhiskAction.get(entityStore, docId.getOrElse(fqnAction.toDocId)) + } // get the whisk action associate with it and inherit the parameters from the package/binding - WhiskAction.get(entityStore, fqnAction.toDocId) map { + action map { _.inherit(resolvedPkg.parameters) } } @@ -578,6 +720,7 @@ object WhiskActionMetaData WhiskActionMetaData.apply, "namespace", "name", + "_id", "exec", "parameters", "limits", @@ -618,14 +761,18 @@ object WhiskActionMetaData * If it's the actual package, use its name directly as the package path name. * While traversing the package bindings, merge the parameters. */ - def resolveActionAndMergeParameters(entityStore: EntityStore, fullyQualifiedName: FullyQualifiedEntityName)( + def resolveActionAndMergeParameters(entityStore: EntityStore, + fullyQualifiedName: FullyQualifiedEntityName, + version: Option[SemVer] = None)( implicit ec: ExecutionContext, transid: TransactionId): Future[WhiskActionMetaData] = { // first check that there is a package to be resolved val entityPath = fullyQualifiedName.path if (entityPath.defaultPackage) { // this is the default package, nothing to resolve - WhiskActionMetaData.get(entityStore, fullyQualifiedName.toDocId) + WhiskActionVersionList.getMatchedDocId(fullyQualifiedName, version, entityStore).flatMap { docId => + WhiskActionMetaData.get(entityStore, docId.getOrElse(fullyQualifiedName.toDocId)) + } } else { // there is a package to be resolved val pkgDocid = fullyQualifiedName.path.toDocId @@ -634,8 +781,11 @@ object WhiskActionMetaData wp flatMap { resolvedPkg => // fully resolved name for the action val fqnAction = resolvedPkg.fullyQualifiedName(withVersion = false).add(actionName) + val action = WhiskActionVersionList.getMatchedDocId(fqnAction, version, entityStore).flatMap { docId => + WhiskActionMetaData.get(entityStore, docId.getOrElse(fqnAction.toDocId)) + } // get the whisk action associate with it and inherit the parameters from the package/binding - WhiskActionMetaData.get(entityStore, fqnAction.toDocId) map { + action map { _.inherit( resolvedPkg.parameters, if (fullyQualifiedName.path.equals(resolvedPkg.fullPath)) None diff --git a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskEntity.scala b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskEntity.scala index a3630a5a330..eeaf9ea8a60 100644 --- a/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskEntity.scala +++ b/common/scala/src/main/scala/org/apache/openwhisk/core/entity/WhiskEntity.scala @@ -58,7 +58,7 @@ abstract class WhiskEntity protected[entity] (en: EntityName, val entityType: St FullyQualifiedEntityName(namespace, en, if (withVersion) Some(version) else None) /** The primary key for the entity in the datastore */ - override final def docid = fullyQualifiedName(false).toDocId + override def docid = fullyQualifiedName(false).toDocId /** * Returns a JSON object with the fields specific to this abstract class. @@ -125,13 +125,14 @@ object WhiskEntity { object WhiskDocumentReader extends DocumentReader { override def read[A](ma: Manifest[A], value: JsValue) = { val doc = ma.runtimeClass match { - case x if x == classOf[WhiskAction] => WhiskAction.serdes.read(value) - case x if x == classOf[WhiskActionMetaData] => WhiskActionMetaData.serdes.read(value) - case x if x == classOf[WhiskPackage] => WhiskPackage.serdes.read(value) - case x if x == classOf[WhiskActivation] => WhiskActivation.serdes.read(value) - case x if x == classOf[WhiskTrigger] => WhiskTrigger.serdes.read(value) - case x if x == classOf[WhiskRule] => WhiskRule.serdes.read(value) - case _ => throw DocumentUnreadable(Messages.corruptedEntity) + case x if x == classOf[WhiskAction] => WhiskAction.serdes.read(value) + case x if x == classOf[WhiskActionMetaData] => WhiskActionMetaData.serdes.read(value) + case x if x == classOf[WhiskPackage] => WhiskPackage.serdes.read(value) + case x if x == classOf[WhiskActivation] => WhiskActivation.serdes.read(value) + case x if x == classOf[WhiskTrigger] => WhiskTrigger.serdes.read(value) + case x if x == classOf[WhiskRule] => WhiskRule.serdes.read(value) + case x if x == classOf[WhiskActionDefaultVersion] => WhiskActionDefaultVersion.serdes.read(value) + case _ => throw DocumentUnreadable(Messages.corruptedEntity) } value.asJsObject.fields.get("entityType").foreach { case JsString(entityType) if (doc.entityType != entityType) => diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala index cc12388026b..4954d030db3 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Actions.scala @@ -19,6 +19,7 @@ package org.apache.openwhisk.core.controller import scala.concurrent.Future import scala.concurrent.duration._ +import scala.language.postfixOps import scala.util.{Failure, Success, Try} import org.apache.kafka.common.errors.RecordTooLargeException import akka.actor.ActorSystem @@ -33,7 +34,14 @@ import org.apache.openwhisk.common.TransactionId import org.apache.openwhisk.core.{FeatureFlags, WhiskConfig} import org.apache.openwhisk.core.controller.RestApiCommons.{ListLimit, ListSkip} import org.apache.openwhisk.core.controller.actions.PostActionActivation -import org.apache.openwhisk.core.database.{ActivationStore, CacheChangeNotification, NoDocumentException} +import org.apache.openwhisk.core.database.{ + ActivationStore, + ArtifactStoreException, + CacheChangeNotification, + DocumentConflictException, + DocumentTypeMismatchException, + NoDocumentException +} import org.apache.openwhisk.core.entitlement._ import org.apache.openwhisk.core.entity._ import org.apache.openwhisk.core.entity.types.EntityStore @@ -108,6 +116,9 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with protected def executeOnly = loadConfigOrThrow[Boolean](ConfigKeys.sharedPackageExecuteOnly) + private val actionMaxVersionLimit = + loadConfigOrThrow[Int](ConfigKeys.actionVersionLimit) + /** Entity normalizer to JSON object. */ import RestApiCommons.emptyEntityToJsObject @@ -209,24 +220,82 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with * - 500 Internal Server Error */ override def create(user: Identity, entityName: FullyQualifiedEntityName)(implicit transid: TransactionId) = { - parameter('overwrite ? false) { overwrite => - entity(as[WhiskActionPut]) { content => - val request = content.resolve(user.namespace) - val check = for { - checkLimits <- checkActionLimits(user, content) - checkAdditionalPrivileges <- entitleReferencedEntities(user, Privilege.READ, request.exec).flatMap(_ => - entitlementProvider.check(user, content.exec)) - } yield (checkAdditionalPrivileges, checkLimits) - - onComplete(check) { - case Success(_) => - putEntity(WhiskAction, entityStore, entityName.toDocId, overwrite, update(user, request) _, () => { - make(user, entityName, request) - }) - case Failure(f) => - super.handleEntitlementFailure(f) + parameter('overwrite ? false, 'deleteOld ? false, 'defaultVersion.as[String] ? "") { + (overwrite, deleteOld, defaultVersion) => + Try { + SemVer(defaultVersion) + } match { + case Success(version) => + onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { + case Success(result) if (result.versionMappings.contains(version)) => + val dv = WhiskActionDefaultVersion(entityName.path, entityName.name, Some(version)) + putEntity( + WhiskActionDefaultVersion, + entityStore, + dv.docid, + true, + (old: WhiskActionDefaultVersion) => + Future.successful(dv.revision[WhiskActionDefaultVersion](old.rev)), + () => Future.successful(dv), + postProcess = Some { version: WhiskActionDefaultVersion => + WhiskActionVersionList.deleteCache(entityName) + complete(OK, version) + }) + case Success(_) => + terminate(NotFound, s"[PUT] entity doesn't has version $version") + case Failure(_) => + terminate(InternalServerError) + } + case Failure(_) => + entity(as[WhiskActionPut]) { content => + val request = content.resolve(user.namespace) + val check = for { + checkLimits <- checkActionLimits(user, content) + checkAdditionalPrivileges <- entitleReferencedEntities(user, Privilege.READ, request.exec).flatMap(_ => + entitlementProvider.check(user, content.exec)) + } yield (checkAdditionalPrivileges, checkLimits) + + onComplete(check) { + case Success(_) => + onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { + case Success(result) if (result.versionMappings.size >= actionMaxVersionLimit && !deleteOld) => + terminate( + Forbidden, + s"[PUT] entity has ${result.versionMappings.size} versions exist which exceed maximum limit, delete one of them before create new one or pass deleteOld=true to delete oldest version automatically") + case Success(result) => + val id = result.matchedDocId(None).getOrElse(entityName.toDocId) + putEntity( + WhiskAction, + entityStore, + id, + true, + update(user, request) _, + () => { + make(user, entityName, request) + }, + postProcess = Some { action: WhiskAction => + // delete oldest version when created successfully + if (result.versionMappings.size >= actionMaxVersionLimit) { + val docid = result.versionMappings.minBy(_._1)._2 + WhiskAction.get(entityStore, docid) flatMap { entity => + WhiskAction.del(entityStore, DocInfo ! (docid.id, entity.rev.rev)).map(_ => entity) + } andThen { + case _ => + WhiskActionVersionList.deleteCache(entityName) + } + } else { + WhiskActionVersionList.deleteCache(entityName) + } + complete(OK, action) + }) + case Failure(f) => + terminate(InternalServerError) + } + case Failure(f) => + super.handleEntitlementFailure(f) + } + } } - } } } @@ -246,10 +315,11 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with parameter( 'blocking ? false, 'result ? false, + 'version.as[SemVer] ?, 'timeout.as[FiniteDuration] ? controllerActivationConfig.maxWaitForBlockingActivation) { - (blocking, result, waitOverride) => + (blocking, result, version, waitOverride) => entity(as[Option[JsObject]]) { payload => - getEntity(WhiskActionMetaData.resolveActionAndMergeParameters(entityStore, entityName), Some { + getEntity(WhiskActionMetaData.resolveActionAndMergeParameters(entityStore, entityName, version), Some { act: WhiskActionMetaData => // resolve the action --- special case for sequences that may contain components with '_' as default package val action = act.resolve(user.namespace) @@ -339,12 +409,98 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with * - 500 Internal Server Error */ override def remove(user: Identity, entityName: FullyQualifiedEntityName)(implicit transid: TransactionId) = { - deleteEntity(WhiskAction, entityStore, entityName.toDocId, (a: WhiskAction) => Future.successful({})) + parameter('version.as[SemVer] ?, 'deleteAll ? false) { (version, deleteAll) => + onComplete(WhiskActionVersionList.get(entityName, entityStore, false)) { + case Success(results) => + version match { + case Some(_) => + val docId = results.matchedDocId(version).getOrElse(entityName.toDocId) + deleteEntity( + WhiskAction, + entityStore, + docId, + (a: WhiskAction) => Future.successful({}), + postProcess = Some { action: WhiskAction => + // when default version is deleted or all versions are deleted, delete the default version entity + if (version == results.defaultVersion || results.versionMappings.size == 1) + deleteDefaultVersion( + WhiskActionDefaultVersion(entityName.path, entityName.name, results.defaultVersion)) + WhiskActionVersionList.deleteCache(entityName) + complete(OK, action) + }) + case None if !deleteAll && results.versionMappings.size > 1 => + terminate( + Forbidden, + s"[DEL] entity version not provided, you need to specify deleteAll=true to delete all versions for action $entityName") + case None => + val fs = + if (results.versionMappings.isEmpty) + Seq(WhiskAction.get(entityStore, entityName.toDocId) flatMap { entity => + WhiskAction.del(entityStore, entity.docinfo).map(_ => entity) + }) + else + results.versionMappings.values.map { docid => + WhiskAction.get(entityStore, docid) flatMap { entity => + WhiskAction.del(entityStore, DocInfo ! (docid.id, entity.rev.rev)).map(_ => entity) + } + } + val deleteFuture = Future.sequence(fs).andThen { + case _ => + WhiskActionVersionList + .deleteCache(entityName) // invalidate version list cache after all deletion completed + deleteDefaultVersion(WhiskActionDefaultVersion( + entityName.path, + entityName.name, + results.defaultVersion)) // delete default version entity since all versions are deleted + } + + onComplete(deleteFuture) { + case Success(entities) => + complete(OK, entities.last) + case Failure(t: NoDocumentException) => + logging.debug(this, s"[DEL] entity does not exist") + terminate(NotFound) + case Failure(t: DocumentConflictException) => + logging.debug(this, s"[DEL] entity conflict: ${t.getMessage}") + terminate(Conflict, conflictMessage) + case Failure(RejectRequest(code, message)) => + logging.debug(this, s"[DEL] entity rejected with code $code: $message") + terminate(code, message) + case Failure(t: DocumentTypeMismatchException) => + logging.debug(this, s"[DEL] entity conformance check failed: ${t.getMessage}") + terminate(Conflict, conformanceMessage) + case Failure(t: ArtifactStoreException) => + logging.error(this, s"[DEL] entity unreadable") + terminate(InternalServerError, t.getMessage) + case Failure(t: Throwable) => + logging.error(this, s"[DEL] entity failed: ${t.getMessage}") + terminate(InternalServerError) + } + } + case Failure(t) => + terminate(InternalServerError, t.getMessage) + } + } + } + + private def deleteDefaultVersion(defaultVersion: WhiskActionDefaultVersion)(implicit transid: TransactionId): Unit = { + WhiskActionDefaultVersion.get(entityStore, defaultVersion.docid) map { entity => + WhiskActionDefaultVersion.del(entityStore, defaultVersion.docid.asDocInfo(entity.rev)) andThen { + case Success(_) => + logging.info(this, s"[DEL] default version for ${defaultVersion.fullyQualifiedName(false)} is deleted") + case Failure(t) => + logging.error( + this, + s"[DEL] failed to delete default version for ${defaultVersion.fullyQualifiedName(false)}, error: $t") + } + } } /** Checks for package binding case. we don't want to allow get for a package binding in shared package */ - private def fetchEntity(entityName: FullyQualifiedEntityName, env: Option[Parameters], code: Boolean)( - implicit transid: TransactionId) = { + private def fetchEntity(entityName: FullyQualifiedEntityName, + env: Option[Parameters], + code: Boolean, + version: Option[SemVer])(implicit transid: TransactionId) = { val resolvedPkg: Future[Either[String, FullyQualifiedEntityName]] = if (entityName.path.defaultPackage) { Future.successful(Right(entityName)) } else { @@ -363,7 +519,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with case Left(f) => terminate(Forbidden, f) case Right(_) => if (code) { - getEntity(WhiskAction.resolveActionAndMergeParameters(entityStore, entityName), Some { + getEntity(WhiskAction.resolveActionAndMergeParameters(entityStore, entityName, version), Some { action: WhiskAction => val mergedAction = env map { action inherit _ @@ -371,7 +527,7 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with complete(OK, mergedAction) }) } else { - getEntity(WhiskActionMetaData.resolveActionAndMergeParameters(entityStore, entityName), Some { + getEntity(WhiskActionMetaData.resolveActionAndMergeParameters(entityStore, entityName, version), Some { action: WhiskActionMetaData => val mergedAction = env map { action inherit _ @@ -396,13 +552,20 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with */ override def fetch(user: Identity, entityName: FullyQualifiedEntityName, env: Option[Parameters])( implicit transid: TransactionId) = { - parameter('code ? true) { code => - //check if execute only is enabled, and if there is a discrepancy between the current user's namespace - //and that of the entity we are trying to fetch - if (executeOnly && user.namespace.name != entityName.namespace) { + parameter('code ? true, 'version.as[SemVer] ?, 'showVersions ? false) { (code, version, showVersions) => + if (showVersions) { + onComplete(WhiskActionVersionList.get(entityName, entityStore)) { + case Success(res) => + complete(OK, res) + case Failure(_) => + terminate(InternalServerError) + } + //check if execute only is enabled, and if there is a discrepancy between the current user's namespace + //and that of the entity we are trying to fetch + } else if (executeOnly && user.namespace.name != entityName.namespace) { terminate(Forbidden, forbiddenGetAction(entityName.path.asString)) } else { - fetchEntity(entityName, env, code) + fetchEntity(entityName, env, code, version) } } } @@ -599,7 +762,6 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with content.version getOrElse action.version.upPatch, content.publish getOrElse action.publish, WhiskActionsApi.amendAnnotations(newAnnotations, exec, create = false)) - .revision[WhiskAction](action.docinfo.rev) } /** @@ -716,13 +878,16 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with } else { // check whether component is a sequence or an atomic action // if the component does not exist, the future will fail with appropriate error - WhiskAction.get(entityStore, resolvedComponent.toDocId) flatMap { wskComponent => - wskComponent.exec match { - case SequenceExec(seqComponents) => - // sequence action, count the number of atomic actions in this sequence - countAtomicActionsAndCheckCycle(origSequence, seqComponents) - case _ => Future successful 1 // atomic action count is one - } + WhiskActionVersionList.getMatchedDocId(resolvedComponent, resolvedComponent.version, entityStore) flatMap { + docId => + WhiskAction.get(entityStore, docId.getOrElse(resolvedComponent.toDocId)) flatMap { wskComponent => + wskComponent.exec match { + case SequenceExec(seqComponents) => + // sequence action, count the number of atomic actions in this sequence + countAtomicActionsAndCheckCycle(origSequence, seqComponents) + case _ => Future successful 1 // atomic action count is one + } + } } } } @@ -759,6 +924,10 @@ trait WhiskActionsApi extends WhiskCollectionAPI with PostActionActivation with /** Custom unmarshaller for query parameters "skip" for "list" operations. */ private implicit val stringToListSkip: Unmarshaller[String, ListSkip] = RestApiCommons.stringToListSkip(collection) + private implicit val stringToSemVer: Unmarshaller[String, SemVer] = Unmarshaller.strict[String, SemVer] { value => + SemVer(value) + } + } private case class TooManyActionsInSequence() extends IllegalArgumentException diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/ApiUtils.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/ApiUtils.scala index ec08f5b3004..5a40c194aca 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/ApiUtils.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/ApiUtils.scala @@ -395,7 +395,7 @@ trait WriteOps extends Directives { onComplete(factory.get(datastore, docid, ignoreMissingAttachment = true) flatMap { entity => confirm(entity) flatMap { case _ => - factory.del(datastore, entity.docinfo) map { _ => + factory.del(datastore, docid.asDocInfo(entity.rev)) map { _ => entity } } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala index c524e9570b1..1e42c98d774 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Packages.scala @@ -87,13 +87,18 @@ trait WhiskPackagesApi extends WhiskCollectionAPI with ReferencedEntities { onComplete(entitlementProvider.check(user, Privilege.READ, referencedentities)) { case Success(_) => - putEntity( - WhiskPackage, - entityStore, - entityName.toDocId, - overwrite, - update(request) _, - () => create(request, entityName)) + onComplete(WhiskActionVersionList.getMatchedDocId(entityName, None, entityStore)) { + case Success(docId) => + putEntity( + WhiskPackage, + entityStore, + docId.getOrElse(entityName.toDocId), + overwrite, + update(request) _, + () => create(request, entityName)) + case Failure(f) => + terminate(InternalServerError) + } case Failure(f) => rewriteEntitlementFailure(f) } @@ -149,14 +154,19 @@ trait WhiskPackagesApi extends WhiskCollectionAPI with ReferencedEntities { case Right(list) if list.nonEmpty && force => Future sequence { list.map(action => { - WhiskAction.get( - entityStore, - wp.fullyQualifiedName(false) - .add(action.fullyQualifiedName(false).name) - .toDocId) flatMap { actionWithRevision => + WhiskAction.get(entityStore, action.docid) flatMap { actionWithRevision => WhiskAction.del(entityStore, actionWithRevision.docinfo) } }) + } andThen { + case _ => + list.foreach { action => + WhiskActionVersionList.deleteCache(action.fullyQualifiedName(false)) + val version = WhiskActionDefaultVersion(action.namespace, action.name, None) + WhiskActionDefaultVersion.get(entityStore, version.docid) foreach { versionWithRevision => + WhiskActionDefaultVersion.del(entityStore, versionWithRevision.docinfo) + } + } } flatMap { _ => Future.successful({}) } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Rules.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Rules.scala index 62b7cacda1a..8a5654e25dd 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Rules.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Rules.scala @@ -85,31 +85,36 @@ trait WhiskRulesApi extends WhiskCollectionAPI with ReferencedEntities { val request = content.resolve(entityName.namespace) onComplete(entitlementProvider.check(user, Privilege.READ, referencedEntities(request))) { case Success(_) => - putEntity( - WhiskRule, - entityStore, - entityName.toDocId, - overwrite, - update(request) _, - () => { - create(request, entityName) - }, - postProcess = Some { rule: WhiskRule => - if (overwrite == true) { - val getRuleWithStatus = getTrigger(rule.trigger) map { trigger => - getStatus(trigger, FullyQualifiedEntityName(rule.namespace, rule.name)) - } map { status => - rule.withStatus(status) - } - - onComplete(getRuleWithStatus) { - case Success(r) => completeAsRuleResponse(rule, r.status) - case Failure(t) => terminate(InternalServerError) - } - } else { - completeAsRuleResponse(rule, Status.ACTIVE) - } - }) + onComplete(WhiskActionVersionList.getMatchedDocId(entityName, None, entityStore)) { + case Success(docId) => + putEntity( + WhiskRule, + entityStore, + docId.getOrElse(entityName.toDocId), + overwrite, + update(request) _, + () => { + create(request, entityName) + }, + postProcess = Some { rule: WhiskRule => + if (overwrite == true) { + val getRuleWithStatus = getTrigger(rule.trigger) map { trigger => + getStatus(trigger, FullyQualifiedEntityName(rule.namespace, rule.name)) + } map { status => + rule.withStatus(status) + } + + onComplete(getRuleWithStatus) { + case Success(r) => completeAsRuleResponse(rule, r.status) + case Failure(t) => terminate(InternalServerError) + } + } else { + completeAsRuleResponse(rule, Status.ACTIVE) + } + }) + case Failure(f) => + terminate(InternalServerError) + } case Failure(f) => handleEntitlementFailure(f) } @@ -408,7 +413,9 @@ trait WhiskRulesApi extends WhiskCollectionAPI with ReferencedEntities { } actionExists <- WhiskAction.resolveAction(entityStore, action) flatMap { resolvedName => - WhiskActionMetaData.get(entityStore, resolvedName.toDocId) + WhiskActionVersionList.getMatchedDocId(resolvedName, resolvedName.version, entityStore).flatMap { docId => + WhiskActionMetaData.get(entityStore, docId.getOrElse(resolvedName.toDocId)) + } } recoverWith { case _: NoDocumentException => Future.failed { diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Triggers.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Triggers.scala index 0c00e3a9b9c..1aee89aa1a6 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Triggers.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/Triggers.scala @@ -20,7 +20,7 @@ package org.apache.openwhisk.core.controller import java.time.{Clock, Instant} import scala.collection.immutable.Map import scala.concurrent.Future -import scala.util.Try +import scala.util.{Failure, Success, Try} import akka.actor.ActorSystem import akka.http.scaladsl.Http import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ @@ -43,6 +43,7 @@ import org.apache.openwhisk.core.entitlement.Collection import org.apache.openwhisk.core.entity._ import org.apache.openwhisk.core.entity.types.EntityStore import org.apache.openwhisk.http.ErrorResponse +import org.apache.openwhisk.http.ErrorResponse.terminate import org.apache.openwhisk.http.Messages import org.apache.openwhisk.core.database.UserContext @@ -91,6 +92,9 @@ trait WhiskTriggersApi extends WhiskCollectionAPI { import RestApiCommons.emptyEntityToJsObject + /** JSON response formatter. */ + import RestApiCommons.jsonDefaultResponsePrinter + /** * Creates or updates trigger if it already exists. The PUT content is deserialized into a WhiskTriggerPut * which is a subset of WhiskTrigger (it eschews the namespace and entity name since the former is derived @@ -107,11 +111,23 @@ trait WhiskTriggersApi extends WhiskCollectionAPI { override def create(user: Identity, entityName: FullyQualifiedEntityName)(implicit transid: TransactionId) = { parameter('overwrite ? false) { overwrite => entity(as[WhiskTriggerPut]) { content => - putEntity(WhiskTrigger, entityStore, entityName.toDocId, overwrite, update(content) _, () => { - create(content, entityName) - }, postProcess = Some { trigger => - completeAsTriggerResponse(trigger) - }) + onComplete(WhiskActionVersionList.getMatchedDocId(entityName, None, entityStore)) { + case Success(docId) => + putEntity( + WhiskTrigger, + entityStore, + docId.getOrElse(entityName.toDocId), + overwrite, + update(content) _, + () => { + create(content, entityName) + }, + postProcess = Some { trigger => + completeAsTriggerResponse(trigger) + }) + case Failure(f) => + terminate(InternalServerError) + } } } } diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/actions/PrimitiveActions.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/actions/PrimitiveActions.scala index 694ef7fa05c..e85786740f7 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/controller/actions/PrimitiveActions.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/controller/actions/PrimitiveActions.scala @@ -183,6 +183,7 @@ protected[actions] trait PrimitiveActions { action.rev, user, activationId, // activation id created here + action.docid, activeAckTopicIndex, waitForResponse.isDefined, args, diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/entitlement/PackageCollection.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/entitlement/PackageCollection.scala index b53dcbc7d37..f541d912dfb 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/entitlement/PackageCollection.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/entitlement/PackageCollection.scala @@ -62,8 +62,10 @@ class PackageCollection(entityStore: EntityStore)(implicit logging: Logging) ext case Privilege.READ => // must determine if this is a public or owned package // or, for a binding, that it references a public or owned package - val docid = FullyQualifiedEntityName(resource.namespace.root.toPath, EntityName(pkgname)).toDocId - checkPackageReadPermission(namespaces, isOwner, docid) + val entityName = FullyQualifiedEntityName(resource.namespace.root.toPath, EntityName(pkgname)) + WhiskActionVersionList.getMatchedDocId(entityName, None, entityStore).flatMap { docId => + checkPackageReadPermission(namespaces, isOwner, docId.getOrElse(entityName.toDocId)) + } case _ => Future.successful(isOwner && allowedEntityRights.contains(right)) } } getOrElse { diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/FPCPoolBalancer.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/FPCPoolBalancer.scala index bf04d1d2a4d..351956fe2bc 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/FPCPoolBalancer.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/FPCPoolBalancer.scala @@ -537,6 +537,7 @@ class FPCPoolBalancer(config: WhiskConfig, val metadata = ExecutableWhiskActionMetaData( WarmUp.warmUpAction.path, WarmUp.warmUpAction.name, + DocId(WarmUp.warmUpAction.asString), CodeExecMetaDataAsString(manifest, false, entryPoint = None)) CreateQueue( WarmUp.warmUpActionIdentity.namespace.name.asString, diff --git a/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/InvokerSupervision.scala b/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/InvokerSupervision.scala index 24b0f3369e2..0e8195c815a 100644 --- a/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/InvokerSupervision.scala +++ b/core/controller/src/main/scala/org/apache/openwhisk/core/loadBalancer/InvokerSupervision.scala @@ -425,6 +425,7 @@ class InvokerActor(invokerInstance: InvokerInstanceId, controllerInstance: Contr user = InvokerPool.healthActionIdentity, // Create a new Activation ID for this activation activationId = new ActivationIdGenerator {}.make(), + actionId = action.docid, rootControllerIndex = controllerInstance, blocking = false, content = None, diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/InvokerHealthManager.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/InvokerHealthManager.scala index 3554e515d66..c0e5af7162f 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/InvokerHealthManager.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/v2/InvokerHealthManager.scala @@ -325,6 +325,7 @@ object InvokerHealthManager { action.rev, healthActionIdentity, ActivationId.generate(), + docInfo.id, ControllerInstanceId("health"), blocking = false, content = None)) diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/ContainerMessageConsumer.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/ContainerMessageConsumer.scala index 416ce50f4d5..593f8a72eaf 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/ContainerMessageConsumer.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/ContainerMessageConsumer.scala @@ -69,7 +69,7 @@ class ContainerMessageConsumer( val createContainer = for { identity <- Identity.get(authStore, EntityName(creation.invocationNamespace)) action <- WhiskAction - .get(entityStore, creation.action.toDocId, creation.revision, fromCache = true) + .get(entityStore, creation.whiskActionMetaData.docId, creation.revision, fromCache = true) } yield { // check action limits before creating container action.limits.checkLimits(identity) diff --git a/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/InvokerReactive.scala b/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/InvokerReactive.scala index 8d821e42202..d67db3121df 100644 --- a/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/InvokerReactive.scala +++ b/core/invoker/src/main/scala/org/apache/openwhisk/core/invoker/InvokerReactive.scala @@ -164,18 +164,17 @@ class InvokerReactive( def handleActivationMessage(msg: ActivationMessage)(implicit transid: TransactionId): Future[Unit] = { val namespace = msg.action.path val name = msg.action.name - val actionid = FullyQualifiedEntityName(namespace, name).toDocId.asDocInfo(msg.revision) val subject = msg.user.subject - logging.debug(this, s"${actionid.id} $subject ${msg.activationId}") + logging.debug(this, s"${msg.actionId} $subject ${msg.activationId}") // caching is enabled since actions have revision id and an updated // action will not hit in the cache due to change in the revision id; // if the doc revision is missing, then bypass cache - if (actionid.rev == DocRevision.empty) logging.warn(this, s"revision was not provided for ${actionid.id}") + if (msg.revision == DocRevision.empty) logging.warn(this, s"revision was not provided for ${msg.actionId}") WhiskAction - .get(entityStore, actionid.id, actionid.rev, fromCache = actionid.rev != DocRevision.empty) + .get(entityStore, msg.actionId, msg.revision, fromCache = msg.revision != DocRevision.empty) .flatMap(action => { // action that exceed the limit cannot be executed. action.limits.checkLimits(msg.user) diff --git a/tests/dat/actions/zippedaction/package-lock.json b/tests/dat/actions/zippedaction/package-lock.json index 4bbddd42a9c..8ca4ddd42d9 100644 --- a/tests/dat/actions/zippedaction/package-lock.json +++ b/tests/dat/actions/zippedaction/package-lock.json @@ -1,8 +1,31 @@ { "name": "test-action", "version": "1.0.0", - "lockfileVersion": 1, + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "name": "test-action", + "version": "1.0.0", + "license": "Apache 2.0", + "dependencies": { + "prog-quote": "2.0.0" + } + }, + "node_modules/prog-quote": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prog-quote/-/prog-quote-2.0.0.tgz", + "integrity": "sha1-TLBMeosV/zu/kxMQxCsBzSjcMB0=", + "dependencies": { + "random-js": "1.0.8" + } + }, + "node_modules/random-js": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/random-js/-/random-js-1.0.8.tgz", + "integrity": "sha1-lo/WiabyXWwKrHZig94vaIycGQo=" + } + }, "dependencies": { "prog-quote": { "version": "2.0.0", diff --git a/tests/src/test/scala/common/WskCliOperations.scala b/tests/src/test/scala/common/WskCliOperations.scala index 925e5d44aac..7be210a2848 100644 --- a/tests/src/test/scala/common/WskCliOperations.scala +++ b/tests/src/test/scala/common/WskCliOperations.scala @@ -206,6 +206,7 @@ class CliActionOperations(override val wsk: RunCliCmd) update: Boolean = false, web: Option[String] = None, websecure: Option[String] = None, + deleteOld: Boolean = true, expectedExitCode: Int = SUCCESS_EXIT)(implicit wp: WskProps): RunResult = { val params = Seq(noun, if (!update) "create" else "update", "--auth", wp.authKey, fqn(name)) ++ { artifact map { Seq(_) } getOrElse Seq.empty @@ -286,6 +287,7 @@ class CliActionOperations(override val wsk: RunCliCmd) parameterFile: Option[String] = None, blocking: Boolean = false, result: Boolean = false, + version: Option[String] = None, expectedExitCode: Int = SUCCESS_EXIT)(implicit wp: WskProps): RunResult = { val params = Seq(noun, "invoke", "--auth", wp.authKey, fqn(name)) ++ { parameters flatMap { p => diff --git a/tests/src/test/scala/common/WskOperations.scala b/tests/src/test/scala/common/WskOperations.scala index 111f3055bb8..217dc5c061a 100644 --- a/tests/src/test/scala/common/WskOperations.scala +++ b/tests/src/test/scala/common/WskOperations.scala @@ -245,6 +245,7 @@ trait ActionOperations extends DeleteFromCollectionOperations with ListOrGetFrom update: Boolean = false, web: Option[String] = None, websecure: Option[String] = None, + deleteOld: Boolean = true, expectedExitCode: Int = SUCCESS_EXIT)(implicit wp: WskProps): RunResult def invoke(name: String, @@ -252,6 +253,7 @@ trait ActionOperations extends DeleteFromCollectionOperations with ListOrGetFrom parameterFile: Option[String] = None, blocking: Boolean = false, result: Boolean = false, + version: Option[String] = None, expectedExitCode: Int = SUCCESS_EXIT)(implicit wp: WskProps): RunResult } diff --git a/tests/src/test/scala/common/rest/WskRestOperations.scala b/tests/src/test/scala/common/rest/WskRestOperations.scala index 6769ddaafa5..7306c7df38a 100644 --- a/tests/src/test/scala/common/rest/WskRestOperations.scala +++ b/tests/src/test/scala/common/rest/WskRestOperations.scala @@ -201,7 +201,7 @@ trait RestDeleteFromCollectionOperations extends DeleteFromCollectionOperations override def delete(name: String, expectedExitCode: Int = OK.intValue)(implicit wp: WskProps): RestResult = { val (ns, entityName) = getNamespaceEntityName(name) val path = Path(s"$basePath/namespaces/$ns/$noun/$entityName") - val resp = requestEntity(DELETE, path)(wp) + val resp = requestEntity(DELETE, path, Map("deleteAll" -> "true"))(wp) val rr = new RestResult(resp.status, getTransactionId(resp), getRespData(resp)) validateStatusCode(expectedExitCode, rr.statusCode.intValue) rr @@ -272,6 +272,7 @@ class RestActionOperations(implicit val actorSystem: ActorSystem) update: Boolean = false, web: Option[String] = None, websecure: Option[String] = None, + deleteOld: Boolean = true, expectedExitCode: Int = OK.intValue)(implicit wp: WskProps): RestResult = { val (namespace, actionName) = getNamespaceEntityName(name) @@ -381,8 +382,9 @@ class RestActionOperations(implicit val actorSystem: ActorSystem) parameterFile: Option[String] = None, blocking: Boolean = false, result: Boolean = false, + version: Option[String] = None, expectedExitCode: Int = Accepted.intValue)(implicit wp: WskProps): RestResult = { - super.invokeAction(name, parameters, parameterFile, blocking, result, expectedExitCode = expectedExitCode) + super.invokeAction(name, parameters, parameterFile, blocking, result, version, expectedExitCode = expectedExitCode) } private def readCodeFromFile(artifact: Option[String]): Option[String] = { @@ -1326,6 +1328,7 @@ trait RunRestCmd extends Matchers with ScalaFutures with SwaggerValidator { parameterFile: Option[String] = None, blocking: Boolean = false, result: Boolean = false, + version: Option[String] = None, web: Boolean = false, expectedExitCode: Int = Accepted.intValue)(implicit wp: WskProps): RestResult = { val (ns, actName) = getNamespaceEntityName(name) @@ -1334,7 +1337,9 @@ trait RunRestCmd extends Matchers with ScalaFutures with SwaggerValidator { if (web) Path(s"$basePath/web/$systemNamespace/$actName.http") else Path(s"$basePath/namespaces/$ns/actions/$actName") - val paramMap = Map("blocking" -> blocking.toString, "result" -> result.toString) + val paramMap = Map("blocking" -> blocking.toString, "result" -> result.toString) ++ version + .map(value => Map("version" -> value)) + .getOrElse(Map.empty) val input = parameterFile map { pf => Some(FileUtils.readFileToString(new File(pf), StandardCharsets.UTF_8)) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/logging/test/DockerToActivationLogStoreTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/logging/test/DockerToActivationLogStoreTests.scala index a5660f7df81..7c40b22eb11 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/logging/test/DockerToActivationLogStoreTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/logging/test/DockerToActivationLogStoreTests.scala @@ -49,7 +49,11 @@ class DockerToActivationLogStoreTests extends FlatSpec with Matchers with WskAct val user = Identity(Subject(), Namespace(EntityName("testSpace"), uuid), BasicAuthenticationAuthKey(uuid, Secret())) val exec = CodeExecAsString(RuntimeManifest("actionKind", ImageName("testImage")), "testCode", None) - val action = ExecutableWhiskAction(user.namespace.name.toPath, EntityName("actionName"), exec) + val action = ExecutableWhiskAction( + user.namespace.name.toPath, + EntityName("actionName"), + DocId("testSpace/actionName@0.0.1"), + exec) val successfulActivation = WhiskActivation( user.namespace.name.toPath, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerPoolTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerPoolTests.scala index eccf04cd784..b4c52e67229 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerPoolTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerPoolTests.scala @@ -84,6 +84,7 @@ class ContainerPoolTests action.rev, Identity(Subject(), Namespace(invocationNamespace, uuid), BasicAuthenticationAuthKey(uuid, Secret())), ActivationId.generate(), + DocId("asd"), ControllerInstanceId("0"), blocking = false, content = None, @@ -94,11 +95,16 @@ class ContainerPoolTests val invocationNamespace = EntityName("invocationSpace") val differentInvocationNamespace = EntityName("invocationSpace2") - val action = ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec) + val action = ExecutableWhiskAction( + EntityPath("actionSpace"), + EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), + exec) val concurrencyEnabled = Option(WhiskProperties.getProperty("whisk.action.concurrency")).exists(_.toBoolean) val concurrentAction = ExecutableWhiskAction( EntityPath("actionSpace"), EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), exec, limits = ActionLimits(concurrency = ConcurrencyLimit(if (concurrencyEnabled) 3 else 1))) val differentAction = action.copy(name = EntityName("actionName2")) @@ -911,6 +917,7 @@ class ContainerPoolTests val action = ExecutableWhiskAction( EntityPath("actionSpace"), EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), exec, limits = ActionLimits(memory = MemoryLimit(memoryLimit))) val run = createRunMessage(action, invocationNamespace) @@ -1001,7 +1008,12 @@ class ContainerPoolObjectTests extends FlatSpec with Matchers with MockFactory { /** Helper to create a new action from String representations */ def createAction(namespace: String = "actionNS", name: String = "actionName", limits: ActionLimits = ActionLimits()) = - ExecutableWhiskAction(EntityPath(namespace), EntityName(name), actionExec, limits = limits) + ExecutableWhiskAction( + EntityPath(namespace), + EntityName(name), + DocId(s"$namespace/$name@0.0.1"), + actionExec, + limits = limits) /** Helper to create WarmedData with sensible defaults */ def warmedData(action: ExecutableWhiskAction = createAction(), diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerProxyTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerProxyTests.scala index 66f7c77392b..d550bea34af 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerProxyTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/test/ContainerProxyTests.scala @@ -84,13 +84,18 @@ class ContainerProxyTests val memoryLimit = 256.MB val invocationNamespace = EntityName("invocationSpace") - val action = ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec) + val action = ExecutableWhiskAction( + EntityPath("actionSpace"), + EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), + exec) val concurrencyEnabled = Option(WhiskProperties.getProperty("whisk.action.concurrency", "false")).exists(_.toBoolean) val testConcurrencyLimit = if (concurrencyEnabled) ConcurrencyLimit(2) else ConcurrencyLimit(1) val concurrentAction = ExecutableWhiskAction( EntityPath("actionSpace"), EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), exec, limits = ActionLimits(concurrency = testConcurrencyLimit)) @@ -122,6 +127,7 @@ class ContainerProxyTests action.rev, Identity(Subject(), Namespace(invocationNamespace, uuid), BasicAuthenticationAuthKey(uuid, Secret())), ActivationId.generate(), + DocId("asd"), ControllerInstanceId("0"), blocking = false, content = Some(activationArguments), @@ -2017,7 +2023,12 @@ class ContainerProxyTests val keyFalsyAnnotation = Parameters(Annotations.ProvideApiKeyAnnotationName, JsFalse) val actionWithFalsyKeyAnnotation = - ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec, annotations = keyFalsyAnnotation) + ExecutableWhiskAction( + EntityPath("actionSpace"), + EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), + exec, + annotations = keyFalsyAnnotation) machine ! Run(actionWithFalsyKeyAnnotation, message) expectMsg(Transition(machine, Started, Running)) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/ActivationClientProxyTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/ActivationClientProxyTests.scala index cf8ef15b2b0..62dc686c496 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/ActivationClientProxyTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/ActivationClientProxyTests.scala @@ -64,9 +64,10 @@ class ActivationClientProxyTests val timeout = 20.seconds val log = logging - + val testNamespace = "actionSpace" + val testActionName = "actionName" val exec = CodeExecAsString(RuntimeManifest("actionKind", ImageName("testImage")), "testCode", None) - val action = ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec) + val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testActionName), DocId(s"$testNamespace/$testActionName@0.0.1"), exec) val fqn = action.fullyQualifiedName(true) val rev = action.rev val schedulerHost = "127.17.0.1" @@ -82,6 +83,7 @@ class ActivationClientProxyTests action.rev, Identity(Subject(), Namespace(invocationNamespace, uuid), BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerPoolTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerPoolTests.scala index e1da77e6323..33303776849 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerPoolTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerPoolTests.scala @@ -103,6 +103,7 @@ class FunctionPullingContainerPoolTests private val exec = CodeExecAsString(RuntimeManifest(actionKind, ImageName("testImage")), "testCode", None) private val memoryLimit = MemoryLimit.STD_MEMORY.toMB.MB private val whiskAction = WhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec) + private val docId = DocId("actionSpace/actionName@0.0.1") private val invocationNamespace = EntityName("invocationSpace") private val schedulerHost = "127.17.0.1" private val rpcPort = 13001 @@ -112,11 +113,13 @@ class FunctionPullingContainerPoolTests EntityName("bigActionName"), exec, limits = ActionLimits(memory = MemoryLimit(memoryLimit * 2))) + private val bigDocId = DocId("actionSpace/actionName@0.0.1") private val execMetadata = CodeExecMetaDataAsString(exec.manifest, entryPoint = exec.entryPoint) private val actionMetaData = WhiskActionMetaData( whiskAction.namespace, whiskAction.name, + docId, execMetadata, whiskAction.parameters, whiskAction.limits, @@ -127,6 +130,7 @@ class FunctionPullingContainerPoolTests WhiskActionMetaData( bigWhiskAction.namespace, bigWhiskAction.name, + bigDocId, execMetadata, bigWhiskAction.parameters, bigWhiskAction.limits, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerProxyTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerProxyTests.scala index 52362256b3d..eb7e43f95d4 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerProxyTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/containerpool/v2/test/FunctionPullingContainerProxyTests.scala @@ -125,7 +125,7 @@ class FunctionPullingContainerProxyTests val memoryLimit = 256.MB - val action = ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec) + val action = ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), DocId("actionSpace/actionName@0.0.1"), exec) val fqn = FullyQualifiedEntityName(action.namespace, action.name, Some(action.version)) @@ -135,6 +135,7 @@ class FunctionPullingContainerProxyTests action.rev, Identity(Subject(), Namespace(invocationNamespace, uuid), BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala index d2bfde2fcb9..b5b1835ed60 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ActionsApiTests.scala @@ -28,6 +28,9 @@ import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport.sprayJsonUnmars import akka.http.scaladsl.server.Route import spray.json._ import spray.json.DefaultJsonProtocol._ +import org.apache.commons.lang3.StringUtils +import org.apache.openwhisk.core.ConfigKeys +import org.apache.openwhisk.core.connector.ActivationMessage import org.apache.openwhisk.core.controller.WhiskActionsApi import org.apache.openwhisk.core.entity._ import org.apache.openwhisk.core.entity.size._ @@ -36,11 +39,12 @@ import org.apache.openwhisk.http.ErrorResponse import org.apache.openwhisk.http.Messages import org.apache.openwhisk.core.database.UserContext import akka.http.scaladsl.model.headers.RawHeader -import org.apache.commons.lang3.StringUtils -import org.apache.openwhisk.core.connector.ActivationMessage import org.apache.openwhisk.core.entity.Attachments.Inline import org.apache.openwhisk.core.entity.test.ExecHelpers import org.scalatest.{FlatSpec, Matchers} +import pureconfig.loadConfigOrThrow + +import scala.language.postfixOps /** * Tests Actions API. @@ -207,6 +211,45 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } + it should "get action by name and version" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + put(entityStore, action) + + Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + val action2 = action.copy(version = action.version.upPatch) + put(entityStore, action2) + WhiskActionVersionList.deleteCache(action2.fullyQualifiedName(false)) + + // get latest version if version is not specified + Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action2) + } + + Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + Get(s"$collectionPath/${action.name}?version=0.0.2") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action2) + } + + Get(s"$collectionPath/${action.name}?version=0.0.3") ~> Route.seal(routes(creds)) ~> check { + status should be(NotFound) + } + } + it should "get action with updated field" in { implicit val tid = transid() @@ -342,6 +385,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { val expectedWhiskActionMetaData = WhiskActionMetaData( action.namespace, action.name, + action.docid, execMetaData, action.parameters, action.limits, @@ -440,6 +484,155 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } + it should "delete action by name and version" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + put(entityStore, action) + + val action2 = action.copy(version = action.version.upPatch) + put(entityStore, action2) + + val action3 = action2.copy(version = action2.version.upPatch) + put(entityStore, action3) + + // delete action@0.0.1, action@0.0.2 and action@0.0.3 should not be deleted + Delete(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + Get(s"/$namespace/${collection.path}/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(NotFound) + } + + Get(s"/$namespace/${collection.path}/${action.name}?version=0.0.2") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action2) + } + + Get(s"/$namespace/${collection.path}/${action.name}?version=0.0.3") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + + // it should return 403 error when try to delete multi versions without specify deleteAll=true + Delete(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(Forbidden) + } + + // it should delete all actions if version is not specified and deleteAll is passed as true + Delete(s"$collectionPath/${action.name}?deleteAll=true") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + + Get(s"/$namespace/${collection.path}/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(NotFound) + } + } + + it should "delete default version if all versions of an action are deleted" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + put(entityStore, action) + + val action2 = action.copy(version = action.version.upPatch) + put(entityStore, action2) + + val action3 = action2.copy(version = action2.version.upPatch) + put(entityStore, action3) + + val content = WhiskActionPut() + + Put(s"$collectionPath/${action.name}?defaultVersion=0.0.2", content) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + + val response = responseAs[WhiskActionDefaultVersion] + checkWhiskEntityResponse( + response, + WhiskActionDefaultVersion(action.namespace, action.name, Some(action2.version))) + } + + // it should return action@0.0.2 + Get(s"/$namespace/${collection.path}/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action2) + } + + // delete all actions + Delete(s"$collectionPath/${action.name}?deleteAll=true") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + + // default version should be deleted and latest version should be used + put(entityStore, action) + put(entityStore, action2) + put(entityStore, action3) + Get(s"/$namespace/${collection.path}/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + + // it should delete all actions if version is not specified and deleteAll is passed as true + Delete(s"$collectionPath/${action.name}?deleteAll=true") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + } + + it should "delete default version if related version of the action is deleted" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + put(entityStore, action) + + val action2 = action.copy(version = action.version.upPatch) + put(entityStore, action2) + + val action3 = action2.copy(version = action2.version.upPatch) + put(entityStore, action3) + + val content = WhiskActionPut() + + Put(s"$collectionPath/${action.name}?defaultVersion=0.0.1", content) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + + val response = responseAs[WhiskActionDefaultVersion] + checkWhiskEntityResponse(response, WhiskActionDefaultVersion(action.namespace, action.name, Some(action.version))) + } + + // it should return action@0.0.1 + Get(s"/$namespace/${collection.path}/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + // delete action@0.0.1 + Delete(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + // default version should be deleted and latest version should be used + Get(s"/$namespace/${collection.path}/${action.name}") ~> Route.seal(routes(creds)) ~> check { + deleteAction(action2.docid) + deleteAction(action3.docid) + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action3) + } + } + it should "report NotFound for delete non existent action" in { implicit val tid = transid() Delete(s"$collectionPath/xyz") ~> Route.seal(routes(creds)) ~> check { @@ -1050,40 +1243,27 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { put(entityStore, action, false) // install the action into the database directly var content = JsObject("exec" -> JsObject("code" -> "".toJson, "kind" -> action.exec.kind.toJson)) + val action2 = action.copy( + version = action.version.upPatch, + annotations = action.annotations ++ Parameters(WhiskAction.execFieldName, action.exec.kind)) Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[WhiskAction] - checkWhiskEntityResponse( - response, - WhiskAction( - action.namespace, - action.name, - action.exec, - action.parameters, - action.limits, - action.version.upPatch, - action.publish, - action.annotations ++ Parameters(WhiskAction.execFieldName, action.exec.kind))) + checkWhiskEntityResponse(response, action2) } content = """{"annotations":[{"key":"a","value":"B"}]}""".parseJson.asJsObject + val action3 = + action2.copy(annotations = action2.annotations ++ Parameters("a", "B"), version = action2.version.upPatch) Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check { deleteAction(action.docid) + deleteAction(action2.docid) + deleteAction(action3.docid) status should be(OK) val response = responseAs[WhiskAction] - checkWhiskEntityResponse( - response, - WhiskAction( - action.namespace, - action.name, - action.exec, - action.parameters, - action.limits, - action.version.upPatch.upPatch, - action.publish, - action.annotations ++ Parameters("a", "B") ++ Parameters(WhiskAction.execFieldName, action.exec.kind))) + checkWhiskEntityResponse(response, action3) } } @@ -1101,9 +1281,12 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { val content = WhiskActionPut(Some(jsDefault(""))) put(entityStore, action, false) + val action2 = action.copy(version = action.version.upPatch) + // create an action sequence Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check { deleteAction(action.docid) + deleteAction(action2.docid) status should be(OK) val response = responseAs[WhiskAction] response.exec.kind should be(NODEJS) @@ -1120,9 +1303,12 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { val content = WhiskActionPut(Some(jsDefault("")), parameters = Some(Parameters("a", "A"))) put(entityStore, action, false) + val action2 = action.copy(version = action.version.upPatch) + // create an action sequence Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check { deleteAction(action.docid) + deleteAction(action2.docid) status should be(OK) val response = responseAs[WhiskAction] response.exec.kind should be(NODEJS) @@ -1258,7 +1444,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { stream.toString should include(s"serving from cache: ${CacheKey(action)}") stream.reset() - // update should invalidate cache + // update should create new cache for new version Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)(transid())) ~> check { status should be(OK) val response = responseAs[WhiskAction] @@ -1275,12 +1461,11 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { action.annotations ++ systemAnnotations(kind))) } stream.toString should include(s"entity exists, will try to update '$action'") - stream.toString should include(s"invalidating ${CacheKey(action)}") - stream.toString should include(s"caching ${CacheKey(action)}") + stream.toString should include(s"caching ${CacheKey(action.copy(version = action.version.upPatch))}") stream.reset() - // delete should invalidate cache - Delete(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)(transid())) ~> check { + // delete should invalidate cache for all versions + Delete(s"$collectionPath/${action.name}?deleteAll=true") ~> Route.seal(routes(creds)(transid())) ~> check { status should be(OK) val response = responseAs[WhiskAction] checkWhiskEntityResponse( @@ -1514,6 +1699,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { // second request should not fetch from cache Get(s"$collectionPath/$name") ~> Route.seal(routes(creds)(transid())) ~> check { + deleteAction(action.docid) status should be(OK) val response = responseAs[WhiskAction] checkWhiskEntityResponse( @@ -1574,6 +1760,8 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { (0 until 5).map { i => Get(s"$collectionPath/$name") ~> Route.seal(routes(creds)(transid())) ~> check { + if (i == 4) + deleteAction(action.docid) status should be(OK) val response = responseAs[WhiskAction] checkWhiskEntityResponse(response, expectedAction) @@ -1640,7 +1828,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { stream.reset() // delete should invalidate cache - Delete(s"$collectionPath/$name") ~> Route.seal(routes(creds)(transid())) ~> check { + Delete(s"$collectionPath/$name?deleteAll=true") ~> Route.seal(routes(creds)(transid())) ~> check { status should be(OK) val response = responseAs[WhiskAction] checkWhiskEntityResponse( @@ -1713,7 +1901,7 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { response.fields("activationId") should not be None } - Delete(s"$collectionPath/${actionOldSchema.name}") ~> Route.seal(routes(creds)) ~> check { + Delete(s"$collectionPath/${actionOldSchema.name}?deleteAll=true") ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[WhiskAction] checkWhiskEntityResponse( @@ -1730,17 +1918,42 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } } - it should "reject put with conflict for pre-existing action" in { + it should "update action with a put" in { implicit val tid = transid() - val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) - val content = WhiskActionPut(Some(action.exec)) + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b"), ActionLimits()) + val content = WhiskActionPut( + Some(jsDefault("_")), + Some(Parameters("x", "X")), + Some( + ActionLimitsOption( + Some(TimeLimit(TimeLimit.MAX_DURATION)), + Some(MemoryLimit(MemoryLimit.MAX_MEMORY)), + Some(LogLimit(LogLimit.MAX_LOGSIZE)), + Some(ConcurrencyLimit(ConcurrencyLimit.MAX_CONCURRENT))))) put(entityStore, action) + + val action2 = action.copy( + exec = content.exec.get, + parameters = content.parameters.get, + version = action.version.upPatch, + annotations = action.annotations ++ systemAnnotations(NODEJS, create = false), + limits = ActionLimits( + content.limits.get.timeout.get, + content.limits.get.memory.get, + content.limits.get.logs.get, + content.limits.get.concurrency.get)) Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { - status should be(Conflict) + deleteAction(action.docid) + deleteAction(action2.docid) + status should be(OK) + val response = responseAs[WhiskAction] + + response.updated should not be action.updated + checkWhiskEntityResponse(response, action2) } } - it should "update action with a put" in { + it should "update default version only if default version is provided" in { implicit val tid = transid() val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b"), ActionLimits()) val content = WhiskActionPut( @@ -1753,26 +1966,68 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { Some(LogLimit(LogLimit.MAX_LOGSIZE)), Some(ConcurrencyLimit(ConcurrencyLimit.MAX_CONCURRENT))))) put(entityStore, action) - Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check { - deleteAction(action.docid) + + val action2 = action.copy( + exec = content.exec.get, + parameters = content.parameters.get, + version = action.version.upPatch, + annotations = action.annotations ++ systemAnnotations(NODEJS, create = false), + limits = ActionLimits( + content.limits.get.timeout.get, + content.limits.get.memory.get, + content.limits.get.logs.get, + content.limits.get.concurrency.get)) + + Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { status should be(OK) val response = responseAs[WhiskAction] response.updated should not be action.updated - checkWhiskEntityResponse( - response, - WhiskAction( - action.namespace, - action.name, - content.exec.get, - content.parameters.get, - ActionLimits( - content.limits.get.timeout.get, - content.limits.get.memory.get, - content.limits.get.logs.get, - content.limits.get.concurrency.get), - version = action.version.upPatch, - annotations = action.annotations ++ systemAnnotations(NODEJS, create = false))) + checkWhiskEntityResponse(response, action2) + } + + // get version 0.0.2 as no default version is set(the latest one will be choose) + Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + checkWhiskEntityResponse(response, action2) + } + + // set default version to 0.0.1 + Put(s"$collectionPath/${action.name}?defaultVersion=0.0.1", content) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskActionDefaultVersion] + checkWhiskEntityResponse(response, WhiskActionDefaultVersion(action.namespace, action.name, Some(action.version))) + } + + // get version 0.0.1 + Get(s"$collectionPath/${action.name}") ~> Route.seal(routes(creds)) ~> check { + deleteAction(action.docid) + deleteAction(action2.docid) + deleteActionDefaultVersion(DocId(s"${action.fullyQualifiedName(false)}/default")) + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + } + + it should "reject update default version if it doesn't exist" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b"), ActionLimits()) + val content = WhiskActionPut( + Some(jsDefault("_")), + Some(Parameters("x", "X")), + Some( + ActionLimitsOption( + Some(TimeLimit(TimeLimit.MAX_DURATION)), + Some(MemoryLimit(MemoryLimit.MAX_MEMORY)), + Some(LogLimit(LogLimit.MAX_LOGSIZE)), + Some(ConcurrencyLimit(ConcurrencyLimit.MAX_CONCURRENT))))) + put(entityStore, action) + + Put(s"$collectionPath/${action.name}?defaultVersion=0.0.2", content) ~> Route.seal(routes(creds)) ~> check { + deleteAction(action.docid) + status should be(NotFound) } } @@ -1781,19 +2036,86 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) val content = WhiskActionPut(parameters = Some(Parameters("x", "X"))) put(entityStore, action) - Put(s"$collectionPath/${action.name}?overwrite=true", content) ~> Route.seal(routes(creds)) ~> check { + val action2 = action.copy( + version = action.version.upPatch, + parameters = content.parameters.get, + annotations = action.annotations ++ systemAnnotations(NODEJS, false)) + Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { deleteAction(action.docid) + deleteAction(action2.docid) status should be(OK) val response = responseAs[WhiskAction] - checkWhiskEntityResponse( - response, - WhiskAction( - action.namespace, - action.name, - action.exec, - content.parameters.get, - version = action.version.upPatch, - annotations = action.annotations ++ systemAnnotations(NODEJS, false))) + checkWhiskEntityResponse(response, action2) + } + } + + it should "create a new action in database instead of replacing old action" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??"), Parameters("x", "b")) + val content = WhiskActionPut(parameters = Some(Parameters("x", "X"))) + put(entityStore, action) + val action2 = action.copy( + parameters = content.parameters.get, + version = action.version.upPatch, + annotations = action.annotations ++ systemAnnotations(NODEJS, false)) + Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + checkWhiskEntityResponse(response, action2) + } + + // the first version should not be replaced + Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + // it should update based on latest version, action2 here + val content2 = WhiskActionPut(annotations = Some(Parameters("x", "X"))) + val action3 = + action2.copy(annotations = action2.annotations ++ content2.annotations, version = action2.version.upPatch) + Put(s"$collectionPath/${action.name}", content2) ~> Route.seal(routes(creds)) ~> check { + deleteAction(action.docid) + deleteAction(action2.docid) + deleteAction(action3.docid) + status should be(OK) + val response = responseAs[WhiskAction] + checkWhiskEntityResponse(response, action3) + } + } + + it should "delete old action if its versions exceed the limit" in { + implicit val tid = transid() + val action = WhiskAction(namespace, aname(), jsDefault("??")) + put(entityStore, action) + var version = action.version + + val limit = loadConfigOrThrow[Int](ConfigKeys.actionVersionLimit) + (1 until limit) foreach { _ => + version = version.upPatch + val actionN = action.copy(version = version) + put(entityStore, actionN) + } + + // the first version should be still there + Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + val response = responseAs[WhiskAction] + response should be(action) + } + + val content = WhiskActionPut(parameters = Some(Parameters("x", "X"))) + Put(s"$collectionPath/${action.name}", content) ~> Route.seal(routes(creds)) ~> check { + status should be(Forbidden) + } + + Put(s"$collectionPath/${action.name}?deleteOld=true", content) ~> Route.seal(routes(creds)) ~> check { + status should be(OK) + } + // the first version should be deleted automatically + Get(s"$collectionPath/${action.name}?version=0.0.1") ~> Route.seal(routes(creds)) ~> check { + status should be(NotFound) } } @@ -2075,6 +2397,8 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } put(entityStore, action) + // delete cache for WhiskActionVersionList + WhiskActionVersionList.deleteCache(action.fullyQualifiedName(false)) Put(s"$collectionPath/${action.name}?overwrite=true", JsObject.empty) ~> Route.seal(routes(creds)) ~> check { status shouldBe BadRequest @@ -2100,9 +2424,12 @@ class ActionsApiTests extends ControllerTestCommon with WhiskActionsApi { } put(entityStore, action) + // delete cache for WhiskActionVersionList + WhiskActionVersionList.deleteCache(action.fullyQualifiedName(false)) Put(s"$collectionPath/${action.name}?overwrite=true", okUpdate) ~> Route.seal(routes(creds)) ~> check { deleteAction(action.docid) + deleteAction(action.copy(version = action.version.upPatch).docid) status shouldBe OK } } finally { diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ControllerTestCommon.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ControllerTestCommon.scala index a9a72f460c4..d58b5bad071 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ControllerTestCommon.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/ControllerTestCommon.scala @@ -94,10 +94,11 @@ protected trait ControllerTestCommon // Used to ignore `updated` field because timestamp is not known before inserting into the DB // If you use this method, test case that checks timestamp must be added val r = response match { - case whiskAction: WhiskAction => whiskAction.copy(updated = expected.updated) - case whiskActionMetaData: WhiskActionMetaData => whiskActionMetaData.copy(updated = expected.updated) - case whiskTrigger: WhiskTrigger => whiskTrigger.copy(updated = expected.updated) - case whiskPackage: WhiskPackage => whiskPackage.copy(updated = expected.updated) + case whiskAction: WhiskAction => whiskAction.copy(updated = expected.updated) + case defaultVersion: WhiskActionDefaultVersion => defaultVersion.copy(updated = expected.updated) + case whiskActionMetaData: WhiskActionMetaData => whiskActionMetaData.copy(updated = expected.updated) + case whiskTrigger: WhiskTrigger => whiskTrigger.copy(updated = expected.updated) + case whiskPackage: WhiskPackage => whiskPackage.copy(updated = expected.updated) } r should be(expected) } @@ -129,6 +130,13 @@ protected trait ControllerTestCommon Await.result(activationStore.get(activationId, context), timeout) } + def deleteActionDefaultVersion(doc: DocId)(implicit transid: TransactionId) = { + Await.result(WhiskActionDefaultVersion.get(entityStore, doc) flatMap { doc => + logging.debug(this, s"deleting ${doc.docinfo}") + WhiskActionDefaultVersion.del(entityStore, doc.docinfo) + }, dbOpTimeout) + } + def storeActivation( activation: WhiskActivation, isBlockingActivation: Boolean, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/SequenceApiTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/SequenceApiTests.scala index 896c14a73c2..02c80bca4a8 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/controller/test/SequenceApiTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/controller/test/SequenceApiTests.scala @@ -65,12 +65,12 @@ class SequenceApiTests extends ControllerTestCommon with WhiskActionsApi { end = Instant.now) putSimpleSequenceInDB(seqName, namespace, Vector(compName1, compName2)) - deleteAction(DocId(s"$namespace/$compName2")) + deleteAction(DocId(s"$namespace/$compName2@0.0.1")) loadBalancer.whiskActivationStub = Some((1.milliseconds, Right(comp1Activation))) Post(s"$collectionPath/$seqName?blocking=true") ~> Route.seal(routes(creds)) ~> check { - deleteAction(DocId(s"$namespace/$seqName")) - deleteAction(DocId(s"$namespace/$compName1")) + deleteAction(DocId(s"$namespace/$seqName@0.0.1")) + deleteAction(DocId(s"$namespace/$compName1@0.0.1")) status should be(BadGateway) val response = responseAs[JsObject] response.fields("response") shouldBe ActivationResponse.applicationError(sequenceComponentNotFound).toExtendedJson @@ -196,7 +196,7 @@ class SequenceApiTests extends ControllerTestCommon with WhiskActionsApi { // update the sequence Put(s"$collectionPath/${seqName.name}?overwrite=true", updatedContent) ~> Route.seal(routes(creds)) ~> check { - deleteAction(DocId(s"$namespace/${seqName.name}")) + deleteAction(DocId(s"$namespace/${seqName.name}@0.0.1")) status should be(BadRequest) responseAs[ErrorResponse].error shouldBe Messages.sequenceIsCyclic } @@ -215,7 +215,7 @@ class SequenceApiTests extends ControllerTestCommon with WhiskActionsApi { // create an action sequence Put(s"$collectionPath/${seqName.name}", content) ~> Route.seal(routes(creds)) ~> check { - deleteAction(DocId(s"$namespace/${seqName.name}")) + deleteAction(DocId(s"$namespace/${seqName.name}@0.0.1")) status should be(OK) } } @@ -246,7 +246,7 @@ class SequenceApiTests extends ControllerTestCommon with WhiskActionsApi { // create an action sequence Put(s"$collectionPath/${seqName.name}", content) ~> Route.seal(routes(creds)) ~> check { - deleteAction(DocId(s"$namespace/${seqName.name}")) + deleteAction(DocId(s"$namespace/${seqName.name}@0.0.1")) status should be(OK) val response = responseAs[String] } @@ -285,7 +285,7 @@ class SequenceApiTests extends ControllerTestCommon with WhiskActionsApi { // update the sequence Put(s"$collectionPath/$pkg/${seqName.name}?overwrite=true", updatedContent) ~> Route.seal(routes(creds)) ~> check { - deleteAction(DocId(s"$namespace/$pkg/${seqName.name}")) + deleteAction(DocId(s"$namespace/$pkg/${seqName.name}@0.0.1")) status should be(BadRequest) responseAs[ErrorResponse].error shouldBe Messages.sequenceIsCyclic } diff --git a/tests/src/test/scala/org/apache/openwhisk/core/database/test/DocumentHandlerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/database/test/DocumentHandlerTests.scala index 725162b4a29..b59f7fd4691 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/database/test/DocumentHandlerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/database/test/DocumentHandlerTests.scala @@ -28,7 +28,7 @@ import spray.json.DefaultJsonProtocol._ import spray.json._ import org.apache.openwhisk.common.TransactionId import org.apache.openwhisk.core.database.SubjectHandler.SubjectView -import org.apache.openwhisk.core.database.WhisksHandler.ROOT_NS +import org.apache.openwhisk.core.database.WhisksHandler.{FULL_NAME, ROOT_NS} import org.apache.openwhisk.core.database._ import org.apache.openwhisk.core.entity._ @@ -48,14 +48,18 @@ class DocumentHandlerTests extends FlatSpec with Matchers with ScalaFutures with it should "return JsObject when namespace is simple name" in { WhisksHandler.computedFields(JsObject(("namespace", JsString("foo")))) shouldBe JsObject((ROOT_NS, JsString("foo"))) - WhisksHandler.computedFields(newRule("foo").toDocumentRecord) shouldBe JsObject((ROOT_NS, JsString("foo"))) + WhisksHandler.computedFields(newRule("foo").toDocumentRecord) shouldBe JsObject( + (ROOT_NS, JsString("foo")), + (FULL_NAME, JsString("foo/foo"))) } it should "return JsObject when namespace is path" in { WhisksHandler.computedFields(JsObject(("namespace", JsString("foo/bar")))) shouldBe JsObject((ROOT_NS, JsString("foo"))) - WhisksHandler.computedFields(newRule("foo/bar").toDocumentRecord) shouldBe JsObject((ROOT_NS, JsString("foo"))) + WhisksHandler.computedFields(newRule("foo/bar").toDocumentRecord) shouldBe JsObject( + (ROOT_NS, JsString("foo")), + (FULL_NAME, JsString("foo/bar/foo"))) } private def newRule(ns: String): WhiskRule = { diff --git a/tests/src/test/scala/org/apache/openwhisk/core/entity/test/SchemaTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/entity/test/SchemaTests.scala index e03989b9dd4..b8f011aec11 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/entity/test/SchemaTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/entity/test/SchemaTests.scala @@ -560,24 +560,27 @@ class SchemaTests extends FlatSpec with BeforeAndAfter with ExecHelpers with Mat } it should "exclude undefined code in whisk action initializer" in { - ExecutableWhiskAction(EntityPath("a"), EntityName("b"), bb("container1")).containerInitializer() shouldBe { + ExecutableWhiskAction(EntityPath("a"), EntityName("b"), DocId("a/b@0.0.1"), bb("container1")) + .containerInitializer() shouldBe { JsObject("name" -> "b".toJson, "binary" -> false.toJson, "main" -> "main".toJson) } - ExecutableWhiskAction(EntityPath("a"), EntityName("b"), bb("container1", "xyz")).containerInitializer() shouldBe { + ExecutableWhiskAction(EntityPath("a"), EntityName("b"), DocId("a/b@0.0.1"), bb("container1", "xyz")) + .containerInitializer() shouldBe { JsObject("name" -> "b".toJson, "binary" -> false.toJson, "main" -> "main".toJson, "code" -> "xyz".toJson) } - ExecutableWhiskAction(EntityPath("a"), EntityName("b"), bb("container1", "", Some("naim"))) + ExecutableWhiskAction(EntityPath("a"), EntityName("b"), DocId("a/b@0.0.1"), bb("container1", "", Some("naim"))) .containerInitializer() shouldBe { JsObject("name" -> "b".toJson, "binary" -> false.toJson, "main" -> "naim".toJson) } } it should "allow of main override in action initializer" in { - ExecutableWhiskAction(EntityPath("a"), EntityName("b"), jsDefault("")).containerInitializer() shouldBe { + ExecutableWhiskAction(EntityPath("a"), EntityName("b"), DocId("a/b@0.0.1"), jsDefault("")) + .containerInitializer() shouldBe { JsObject("name" -> "b".toJson, "binary" -> false.toJson, "code" -> JsString.empty, "main" -> "main".toJson) } - ExecutableWhiskAction(EntityPath("a"), EntityName("b"), jsDefault("", Some("bar"))) + ExecutableWhiskAction(EntityPath("a"), EntityName("b"), DocId("a/b@0.0.1"), jsDefault("", Some("bar"))) .containerInitializer() shouldBe { JsObject("name" -> "b".toJson, "binary" -> false.toJson, "code" -> JsString.empty, "main" -> "bar".toJson) } @@ -592,7 +595,8 @@ class SchemaTests extends FlatSpec with BeforeAndAfter with ExecHelpers with Mat "E" -> JsArray(JsString("a")), "F" -> JsObject("a" -> JsFalse)) - ExecutableWhiskAction(EntityPath("a"), EntityName("b"), bb("container1")).containerInitializer(env) shouldBe { + ExecutableWhiskAction(EntityPath("a"), EntityName("b"), DocId("a/b@0.0.1"), bb("container1")) + .containerInitializer(env) shouldBe { JsObject( "name" -> "b".toJson, "binary" -> false.toJson, @@ -619,7 +623,11 @@ class SchemaTests extends FlatSpec with BeforeAndAfter with ExecHelpers with Mat it should "compare as equal two executable actions even if their revision does not match" in { val exec = CodeExecAsString(RuntimeManifest("actionKind", ImageName("testImage")), "testCode", None) - val actionA = ExecutableWhiskAction(EntityPath("actionSpace"), EntityName("actionName"), exec) + val actionA = ExecutableWhiskAction( + EntityPath("actionSpace"), + EntityName("actionName"), + DocId("actionSpace/actionName@0.0.1"), + exec) val actionB = actionA.copy() val actionC = actionA.copy() actionC.revision(DocRevision("2")) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/invoker/test/ContainerMessageConsumerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/invoker/test/ContainerMessageConsumerTests.scala index aa2db7c2188..d22f15b4175 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/invoker/test/ContainerMessageConsumerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/invoker/test/ContainerMessageConsumerTests.scala @@ -164,6 +164,7 @@ class ContainerMessageConsumerTests sendAckToScheduler(producer)) val exec = CodeExecAsString(RuntimeManifest("nodejs:14", ImageName("testImage")), "testCode", None) + val docId = DocId("testns/testAction@0.0.1") val action = WhiskAction(EntityPath("testns"), EntityName("testAction"), exec, limits = ActionLimits(TimeLimit(1.minute))) put(entityStore, action) @@ -173,6 +174,7 @@ class ContainerMessageConsumerTests WhiskActionMetaData( action.namespace, action.name, + docId, execMetadata, action.parameters, action.limits, @@ -219,6 +221,7 @@ class ContainerMessageConsumerTests sendAckToScheduler(ackConsumer.getProducer())) val exec = CodeExecAsString(RuntimeManifest("nodejs:14", ImageName("testImage")), "testCode", None) + val docId = DocId("testns/testAction2@0.0.1") val whiskAction = WhiskAction(EntityPath("testns"), EntityName("testAction2"), exec, limits = ActionLimits(TimeLimit(1.minute))) val execMetadata = @@ -227,6 +230,7 @@ class ContainerMessageConsumerTests WhiskActionMetaData( whiskAction.namespace, whiskAction.name, + docId, execMetadata, whiskAction.parameters, whiskAction.limits, @@ -301,7 +305,7 @@ class ContainerMessageConsumerTests WarmUp.warmUpAction.name, exec, limits = ActionLimits(TimeLimit(1.minute))) - val doc = put(entityStore, action) + val docId = DocId(action.fullyQualifiedName(true).asString) val execMetadata = CodeExecMetaDataAsString(exec.manifest, entryPoint = exec.entryPoint) @@ -309,6 +313,7 @@ class ContainerMessageConsumerTests WhiskActionMetaData( action.namespace, action.name, + docId, execMetadata, action.parameters, action.limits, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/InvokerSupervisionTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/InvokerSupervisionTests.scala index 2cf0ef3f3dc..3a13d0141b5 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/InvokerSupervisionTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/InvokerSupervisionTests.scala @@ -187,6 +187,7 @@ class InvokerSupervisionTests Namespace(EntityName("unhealthyInvokerCheck"), uuid), BasicAuthenticationAuthKey(uuid, Secret())), activationId = new ActivationIdGenerator {}.make(), + DocId("asd"), rootControllerIndex = ControllerInstanceId("0"), blocking = false, content = None, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/ShardingContainerPoolBalancerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/ShardingContainerPoolBalancerTests.scala index 2fb70088c9b..866bea9a33a 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/ShardingContainerPoolBalancerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/loadBalancer/test/ShardingContainerPoolBalancerTests.scala @@ -34,8 +34,11 @@ import org.scalatest.Matchers import scala.concurrent.Await import scala.concurrent.Future import scala.concurrent.duration._ -import org.apache.openwhisk.common.{InvokerHealth, Logging, NestedSemaphore, TransactionId} -import org.apache.openwhisk.core.entity.FullyQualifiedEntityName +import org.apache.openwhisk.common.Logging +import org.apache.openwhisk.common.NestedSemaphore +import org.apache.openwhisk.common.InvokerHealth +import org.apache.openwhisk.core.entity._ +import org.apache.openwhisk.common.TransactionId import org.apache.openwhisk.core.WhiskConfig import org.apache.openwhisk.core.connector.{ ActivationMessage, @@ -61,7 +64,6 @@ import org.apache.openwhisk.core.entity.Subject import org.apache.openwhisk.core.entity.UUID import org.apache.openwhisk.core.entity.WhiskActionMetaData import org.apache.openwhisk.core.entity.test.ExecHelpers -import org.apache.openwhisk.core.entity.ByteSize import org.apache.openwhisk.core.entity.size._ import org.apache.openwhisk.core.entity.test.ExecHelpers import org.apache.openwhisk.core.loadBalancer.FeedFactory @@ -418,6 +420,7 @@ class ShardingContainerPoolBalancerTests WhiskActionMetaData( namespace, name, + DocId(s"$namespace/$name@0.0.1"), jsMetaData(Some("jsMain"), false), limits = actionLimits(actionMem, concurrency)) val maxContainers = invokerMem.toMB.toInt / actionMetaData.limits.memory.megabytes @@ -502,6 +505,7 @@ class ShardingContainerPoolBalancerTests actionMetaData.rev, Identity(Subject(), Namespace(invocationNamespace, uuid), BasicAuthenticationAuthKey(uuid, Secret())), aid, + DocId("asd"), ControllerInstanceId("0"), blocking = false, content = None, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/ContainerManagerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/ContainerManagerTests.scala index 2581bcfefa9..59b70bdf79c 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/ContainerManagerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/ContainerManagerTests.scala @@ -93,12 +93,14 @@ class ContainerManagerTests val resourcesStrictPolicy = false val exec = CodeExecAsString(RuntimeManifest("actionKind", ImageName("testImage")), "testCode", None) - val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), exec) + val docId = DocId(s"$testNamespace/$testAction@0.0.1") + val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), docId, exec) val execMetadata = CodeExecMetaDataAsString(exec.manifest, entryPoint = exec.entryPoint) val actionMetadata = WhiskActionMetaData( action.namespace, action.name, + docId, execMetadata, action.parameters, action.limits, @@ -973,12 +975,14 @@ class ContainerManagerTests ContainerManager.props(factory(mockJobManager), mockMessaging(), testsid, mockEtcd, config, mockWatcher.ref)) val exec = BlackBoxExec(ExecManifest.ImageName("image"), None, None, native = false, binary = false) - val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), exec) + val docId = DocId(s"$testNamespace/$testAction@0.0.1") + val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), docId, exec) val execMetadata = BlackBoxExecMetaData(exec.image, exec.entryPoint, exec.native, exec.binary) val actionMetadata = WhiskActionMetaData( action.namespace, action.name, + docId, execMetadata, action.parameters, action.limits, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/CreationJobManagerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/CreationJobManagerTests.scala index 61e8199b7ca..def68ba24d3 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/CreationJobManagerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/container/test/CreationJobManagerTests.scala @@ -68,7 +68,7 @@ class CreationJobManagerTests val schedulerHost = "127.17.0.1" val rpcPort = 13001 val exec = CodeExecAsString(RuntimeManifest("actionKind", ImageName("testImage")), "testCode", None) - val execAction = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), exec) + val execAction = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), DocId(s"$testNamespace/$testAction@0.0.1"), exec) val execMetadata = CodeExecMetaDataAsString(RuntimeManifest(execAction.exec.kind, ImageName("test")), entryPoint = Some("test")) val revision = DocRevision("1-testRev") @@ -76,6 +76,7 @@ class CreationJobManagerTests WhiskActionMetaData( execAction.namespace, execAction.name, + execAction.docId, execMetadata, execAction.parameters, execAction.limits, @@ -342,6 +343,7 @@ class CreationJobManagerTests val actionMetaData = WhiskActionMetaData( execAction.namespace, execAction.name, + execAction.docId, execMetadata, execAction.parameters, execAction.limits, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/ActivationServiceImplTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/ActivationServiceImplTests.scala index b2042f256c9..d6850baa531 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/ActivationServiceImplTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/ActivationServiceImplTests.scala @@ -84,6 +84,7 @@ class ActivationServiceImplTests BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + DocId(s"$testEntityPath/$testEntityName@0.0.1"), ControllerInstanceId("0"), blocking = false, content = None) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/CommonVariable.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/CommonVariable.scala index 04f0e818013..f397dff00ad 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/CommonVariable.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/grpc/test/CommonVariable.scala @@ -29,12 +29,17 @@ trait CommonVariable { val testEntityName = EntityName(testAction) val testDocRevision = DocRevision("1-test-revision") val testContainerId = "fakeContainerId" - val semVer = SemVer(0, 1, 1) + val semVer = SemVer(0, 0, 1) val testVersion = Some(semVer) val testFQN = FullyQualifiedEntityName(testEntityPath, testEntityName, testVersion) val testExec = CodeExecAsString(RuntimeManifest("nodejs:14", ImageName("testImage")), "testCode", None) val testExecMetadata = CodeExecMetaDataAsString(testExec.manifest, entryPoint = testExec.entryPoint) val testActionMetaData = - WhiskActionMetaData(testEntityPath, testEntityName, testExecMetadata, version = semVer) + WhiskActionMetaData( + testEntityPath, + testEntityName, + DocId(s"$testEntityPath/$testEntityName@$semVer"), + testExecMetadata, + version = semVer) } diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala index 4f6d33e2756..e05247b34a2 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/ElasticSearchDurationCheckerTests.scala @@ -137,6 +137,7 @@ class ElasticSearchDurationCheckerTests WhiskActionMetaData( EntityPath(namespace), EntityName(actionName), + DocId(s"$namespace/$actionName@0.0.1"), jsMetaData(Some("jsMain"), binary = false), limits = actionLimits(actionMem, concurrency)) @@ -208,6 +209,7 @@ class ElasticSearchDurationCheckerTests WhiskActionMetaData( EntityPath(s"$namespace/$packageName"), EntityName(actionName), + DocId(s"$namespace/$packageName/$actionName@0.0.1"), jsMetaData(Some("jsMain"), binary = false), limits = actionLimits(actionMem, concurrency)) @@ -286,6 +288,7 @@ class ElasticSearchDurationCheckerTests WhiskActionMetaData( EntityPath(s"$namespace/$boundPackageName"), EntityName(actionName), + DocId(s"$namespace/$boundPackageName/$actionName@0.0.1"), jsMetaData(Some("jsMain"), binary = false), limits = actionLimits(actionMem, concurrency), binding = Some(EntityPath(s"$namespace/$packageName"))) @@ -363,6 +366,7 @@ class ElasticSearchDurationCheckerTests WhiskActionMetaData( EntityPath(s"$namespace"), EntityName(actionName), + DocId(s"$namespace/$actionName@0.0.1"), jsMetaData(Some("jsMain"), binary = false), limits = actionLimits(actionMem, concurrency)) @@ -383,6 +387,7 @@ class ElasticSearchDurationCheckerTests WhiskActionMetaData( EntityPath(s"$namespace/$boundPackageName"), EntityName(actionName), + DocId(s"$namespace/$boundPackageName/$actionName@0.0.1"), jsMetaData(Some("jsMain"), false), limits = actionLimits(actionMem, concurrency), binding = Some(EntityPath(s"${namespace}/${packageName}"))) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala index c2c5ae34003..8e31749f30a 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTests.scala @@ -1342,6 +1342,7 @@ class MemoryQueueTests WhiskActionMetaData( action.namespace, action.name, + action.docId, execMetadata, action.parameters, action.limits, diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala index d793c58486f..81a3f63a0a7 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/MemoryQueueTestsFixture.scala @@ -91,13 +91,14 @@ class MemoryQueueTestsFixture val fqn = FullyQualifiedEntityName(EntityPath(testNamespace), EntityName(testAction), Some(SemVer(0, 0, 1))) val revision = DocRevision("1-testRev") val exec = CodeExecAsString(RuntimeManifest("actionKind", ImageName("testImage")), "testCode", None) - val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), exec) + val action = ExecutableWhiskAction(EntityPath(testNamespace), EntityName(testAction), DocId(s"$testNamespace/$testAction@0.0.1"), exec) val execMetadata = CodeExecMetaDataAsString(RuntimeManifest(action.exec.kind, ImageName("test")), entryPoint = Some("test")) val actionMetadata = WhiskActionMetaData( action.namespace, action.name, + action.docId, execMetadata, action.parameters, action.limits, @@ -125,6 +126,7 @@ class MemoryQueueTestsFixture BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) @@ -301,6 +303,7 @@ class MemoryQueueTestsFixture BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) diff --git a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/QueueManagerTests.scala b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/QueueManagerTests.scala index 5baa97ebc82..35f38d99c61 100644 --- a/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/QueueManagerTests.scala +++ b/tests/src/test/scala/org/apache/openwhisk/core/scheduler/queue/test/QueueManagerTests.scala @@ -84,7 +84,7 @@ class QueueManagerTests val messageTransId = TransactionId(TransactionId.testing.meta.id) val uuid = UUID() - val action = ExecutableWhiskAction(testEntityPath, testEntityName, testExec) + val action = ExecutableWhiskAction(testEntityPath, testEntityName, DocId(s"${testEntityPath.namespace}/${testEntityName.name}@0.0.1"), testExec) val testLeaderKey = QueueKeys.queue(testInvocationNamespace, action.fullyQualifiedName(false), true) val activationMessage = ActivationMessage( @@ -97,6 +97,7 @@ class QueueManagerTests BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) @@ -143,6 +144,7 @@ class QueueManagerTests WhiskActionMetaData( action.namespace, action.name, + action.docid, exec, action.parameters, action.limits, @@ -458,6 +460,7 @@ class QueueManagerTests BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) @@ -543,6 +546,7 @@ class QueueManagerTests BasicAuthenticationAuthKey(uuid, Secret()), Set.empty), ActivationId.generate(), + action.docId, ControllerInstanceId("0"), blocking = false, content = None) @@ -1108,7 +1112,7 @@ class QueueManagerTests val watcher = TestProbe() val warmUpActionMetaData = - WhiskActionMetaData(warmUpAction.namespace.toPath, warmUpAction.name, testExecMetadata, version = semVer) + WhiskActionMetaData(warmUpAction.namespace.toPath, warmUpAction.name, DocId(s"${warmUpAction.namespace}/${warmUpAction.name}@0.0.1"), testExecMetadata, version = semVer) val warmUpQueueCreationMessage = CreateQueue(warmUpAction.namespace.toString, warmUpAction, testDocRevision, warmUpActionMetaData) diff --git a/tests/src/test/scala/system/basic/WskActionTests.scala b/tests/src/test/scala/system/basic/WskActionTests.scala index a7a5cd89a66..98681d1737a 100644 --- a/tests/src/test/scala/system/basic/WskActionTests.scala +++ b/tests/src/test/scala/system/basic/WskActionTests.scala @@ -20,6 +20,7 @@ package system.basic import java.io.File import java.nio.charset.StandardCharsets +import akka.http.scaladsl.model.StatusCodes.NotFound import org.junit.runner.RunWith import org.scalatest.junit.JUnitRunner import common._ @@ -51,6 +52,37 @@ class WskActionTests extends TestHelpers with WskTestHelpers with JsHelpers with } } + it should "save multi versions for an action and invoke them" in withAssetCleaner(wskprops) { (wp, assetHelper) => + val name = "multiVersion" + assetHelper.withCleaner(wsk.action, name) { (action, _) => + action.create(name, Some(TestUtils.getTestActionFilename("hello.js"))) + action.create(name, Some(TestUtils.getTestActionFilename("echo.js"))) + } + + // invoke the default version + var run = wsk.action.invoke(name, Map("payload" -> "world".toJson)) + withActivation(wsk.activation, run) { activation => + activation.response.status shouldBe "success" + activation.response.result shouldBe Some(JsObject("payload" -> "world".toJson)) + activation.logs.get.mkString(" ") shouldBe empty + } + + // invoke the first version + run = wsk.action.invoke(name, Map("payload" -> "world".toJson), version = Some("0.0.1")) + withActivation(wsk.activation, run) { activation => + activation.response.status shouldBe "success" + activation.response.result shouldBe Some(JsObject("payload" -> "hello, world!".toJson)) + activation.logs.get.mkString(" ") should include(s"hello, world!") + } + + // invoke a non-exist version + wsk.action.invoke( + name, + Map("payload" -> "world".toJson), + version = Some("0.0.3"), + expectedExitCode = NotFound.intValue) + } + it should "invoke an action returning a promise" in withAssetCleaner(wskprops) { (wp, assetHelper) => val name = "hello promise" assetHelper.withCleaner(wsk.action, name) { (action, _) => diff --git a/tests/src/test/scala/system/basic/WskRestBasicTests.scala b/tests/src/test/scala/system/basic/WskRestBasicTests.scala index 134808928ca..abb98ce8cba 100644 --- a/tests/src/test/scala/system/basic/WskRestBasicTests.scala +++ b/tests/src/test/scala/system/basic/WskRestBasicTests.scala @@ -282,19 +282,6 @@ class WskRestBasicTests extends TestHelpers with WskTestHelpers with WskActorSys actions.exists(action => RestResult.getField(action, "name") == name) shouldBe true } - it should "reject create of an action that already exists" in withAssetCleaner(wskprops) { - val name = "dupeAction" - val file = Some(TestUtils.getTestActionFilename("echo.js")) - - (wp, assetHelper) => - assetHelper.withCleaner(wsk.action, name) { (action, _) => - action.create(name, file) - } - - val stderr = wsk.action.create(name, file, expectedExitCode = Conflict.intValue).stderr - stderr should include("resource already exists") - } - it should "reject delete of action that does not exist" in { val name = "nonexistentAction" val stderr = wsk.action.delete(name, expectedExitCode = NotFound.intValue).stderr diff --git a/tools/migrate/migrate_to_new_id.py b/tools/migrate/migrate_to_new_id.py new file mode 100644 index 00000000000..06e2a5edc5c --- /dev/null +++ b/tools/migrate/migrate_to_new_id.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +""" +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +""" + +import argparse +import couchdb.client +import requests +from urllib import quote_plus + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Utility to migrate actions with new ids.") + parser.add_argument("--dbUrl", required=True, help="Server URL of the database. E.g. 'https://xxx:yyy@domain.couch.com:443'") + parser.add_argument("--dbName", required=True, help="Name of the Database of the actions to be migration.") + parser.add_argument("--docsPerRequest", type=int, default=200, help="Number of documents handled on each CouchDb Request. Default is 200.") + + args = parser.parse_args() + + db = couchdb.client.Server(args.dbUrl)[args.dbName] + start = 0 + actions = db.view("whisks.v2.1.0/actions", skip=start, limit=args.docsPerRequest, reduce=False) + + while len(actions) != 0: + for action in actions: + actionName = "%s/%s" % (action.value["namespace"], action.value["name"]) + newId = "%s@%s" % (actionName, action.value["version"]) + # this action is using old style id, copy it with new id which append `@version` to it + if(action.id != newId): + print("Copy %s to %s:...........\n" % (action.id, newId)) + url = "%s/%s/%s" % (args.dbUrl, args.dbName, quote_plus(actionName)) + headers = {"Content-Type": "application/json", "Destination": newId} + res = requests.request("COPY", url, headers = headers) + print("Copying result is %s\n" % res.content) + else: + print("Action %s is already using new style id, skip it\n" % action.id) + + start = start + args.docsPerRequest + actions = db.view("whisks.v2.1.0/actions", skip=start, limit=args.docsPerRequest, reduce=False)