Domain APIs + dynamicEntity.definitions_map.cache.ttl.seconds

obp-api/src/main/resources/props/sample.props.template

@@ -94,6 +94,10 @@ starConnector_supported_types=mapped,internal
 ## DynamicEntity cache time-to-live in seconds, default is 30, the value is 0 at test environment
 ## no 0 value will cause new dynamic entity will be shown after that seconds
 dynamicEntity.cache.ttl.seconds=30
+## How long each node keeps the parsed map of every Dynamic Entity definition, which the access and
+## field checks consult many times per request. A definition created, updated or deleted on this node
+## is seen at once; on another node, within this many seconds. Default 300. Not kept in test mode.
+#dynamicEntity.definitions_map.cache.ttl.seconds=300
 ## no 0 value will cause new dynamic endpoints will be shown after that seconds, default is 32, the value is 0 at test environment
 ## DynamicEndpoint cache time-to-live in seconds, default is 0, should set a no 0 value at product environment
 dynamicEndpoint.cache.ttl.seconds=32

obp-api/src/main/scala/code/api/dynamic/domainapi/DomainApiPaths.scala

@@ -31,8 +31,10 @@ import cats.effect.IO
 import cats.effect.unsafe.implicits.global
 import code.api.Constant.ApiPathZero
 import code.api.berlin.group.ConstantsBG
-import code.api.util.APIUtil.ResourceDoc
+import code.api.util.APIUtil.ResourceDoc.isPathVariable
 import code.domainapi.DomainApiRoute
+import code.dynamicEntity.DynamicEntityProvider
+import code.dynamicResourceDoc.DynamicResourceDocProvider
 import com.openbankproject.commons.util.{ApiShortVersions, ApiStandards, ApiVersion}
 import org.json4s.JsonAST.{JObject, JValue}
 
@@ -78,7 +80,7 @@ object DomainApiPaths {
   /**
    * Path segments a Dynamic Entity URL already gives a meaning to right after the space, and the names of
    * the Domain API's own documentation. A Dynamic Resource Doc whose path starts with one of them would be
-   * hidden under a Domain API, so it counts as a clash.
+   * hidden under a Domain API, so it is refused, and so is a Dynamic Entity named one of them.
    */
   val reservedUnderBasePath: Set[String] = Set("my", "public", "community", "openapi.json", "openapi.yaml")
 
@@ -145,30 +147,92 @@ object DomainApiPaths {
     }
   }
 
-  /** A path template with each placeholder (an all-capitals segment) reduced to one form, for comparison. */
-  private def templateKey(path: String): String =
-    path.split("/").filter(_.nonEmpty).map(s => if (s.matches("[A-Z][A-Z0-9_]*")) "{}" else s).mkString("/", "/", "")
+  /** This is one Dynamic Resource Doc as the path rules below see it. */
+  case class ResourceDocPath(dynamicResourceDocId: Option[String], verb: String, path: String, name: String) {
+    def segments: List[String] = path.split("/").filter(_.nonEmpty).toList
+    def describe: String = s"${verb.toUpperCase} $path ($name)"
+  }
+
+  /**
+   * Two paths are ambiguous when one request could match both: they have as many segments, and at each
+   * position the segments are equal or at least one of them is a path variable (an all-capitals segment).
+   */
+  def ambiguous(a: List[String], b: List[String]): Boolean =
+    a.length == b.length && a.zip(b).forall { case (x, y) => x == y || isPathVariable(x) || isPathVariable(y) }
 
   /**
-   * The verb and path pairs that more than one endpoint of the space would publish, and the Dynamic
-   * Resource Docs whose path starts with a segment a Dynamic Entity URL or the documentation already uses.
-   * Each is described for the person who has to resolve it.
+   * This lists why a Dynamic Resource Doc's path would be ambiguous in its space, given the space's
+   * Dynamic Entity names and its other Dynamic Resource Docs; empty when it is not.
+   *
+   * The Dynamic Entities and Dynamic Resource Docs of one space share one set of paths when a Domain API
+   * publishes the space under its base path, where a Dynamic Entity owns every path that starts with its
+   * name. The rules hold in every space, with or without a Domain API, so that one can be registered over
+   * any space at any time. A path may not start with a path variable (it would match every entity name),
+   * with a segment a Dynamic Entity URL or the Domain API's documentation already uses, or with the name
+   * of one of the space's Dynamic Entities, and no other doc of the same verb may match a request it
+   * matches. A doc is never compared with itself (the same dynamicResourceDocId).
    */
-  def clashes(space: String, docs: List[ResourceDoc]): List[String] = {
-    val published = docs.flatMap(doc => publishedPath(space, doc.requestUrl).map(path => (doc, path)))
-    val duplicates = published
-      .groupBy { case (doc, path) => (doc.requestVerb.toUpperCase, templateKey(path)) }
-      .collect { case ((verb, key), entries) if entries.length > 1 =>
-        s"$verb $key (${entries.map(_._1.partialFunctionName).sorted.mkString(", ")})"
-      }.toList
-    val hidden = published.collect {
-      case (doc, path) if doc.requestUrl.contains(s"/$dynamicResourceDocSegment/") &&
-        reservedUnderBasePath.contains(path.split("/").filter(_.nonEmpty).headOption.getOrElse("")) =>
-        s"${doc.requestVerb.toUpperCase} $path (${doc.partialFunctionName}) starts with a reserved segment"
+  def resourceDocAmbiguities(doc: ResourceDocPath, entityNames: List[String], otherDocs: List[ResourceDocPath]): List[String] =
+    doc.segments match {
+      case Nil => List(s"${doc.describe} has no path segment")
+      case first :: _ =>
+        val variableFirst =
+          if (isPathVariable(first)) List(s"${doc.describe} starts with the path variable $first, which would also match every Dynamic Entity name") else Nil
+        val reserved =
+          if (reservedUnderBasePath.contains(first)) List(s"${doc.describe} starts with $first, which a Dynamic Entity URL or a Domain API's documentation already uses") else Nil
+        val entities = entityNames.filter(_.equalsIgnoreCase(first))
+          .map(entityName => s"${doc.describe} starts with $first, the name of the Dynamic Entity $entityName")
+        val docs = otherDocs
+          .filterNot(other => doc.dynamicResourceDocId.isDefined && other.dynamicResourceDocId == doc.dynamicResourceDocId)
+          .filter(other => other.verb.equalsIgnoreCase(doc.verb) && ambiguous(other.segments, doc.segments))
+          .map(other => s"${doc.describe} and ${other.describe} would both match one request")
+        variableFirst ++ reserved ++ entities ++ docs
     }
-    (duplicates ++ hidden).sorted
+
+  /**
+   * This lists why a Dynamic Entity name would be ambiguous in its space, given the space's Dynamic
+   * Resource Docs: a doc whose path starts with the name (compared ignoring case), or a name that is a
+   * segment a Dynamic Entity URL or a Domain API's documentation already uses. Empty when it is neither.
+   */
+  def entityNameAmbiguities(entityName: String, docs: List[ResourceDocPath]): List[String] = {
+    val reserved =
+      if (reservedUnderBasePath.contains(entityName.toLowerCase)) List(s"the Dynamic Entity name $entityName is a segment a Dynamic Entity URL or a Domain API's documentation already uses") else Nil
+    reserved ++ docs.filter(_.segments.headOption.exists(_.equalsIgnoreCase(entityName)))
+      .map(doc => s"${doc.describe} starts with $entityName, the name of the Dynamic Entity")
+  }
+
+  /**
+   * This lists every ambiguity among a space's Dynamic Entities and Dynamic Resource Docs, each pair once.
+   * Writes are checked one at a time, so this finds only what predates the rules; a Domain API is
+   * refused over a space while the list is not empty.
+   */
+  def ambiguitiesInSpace(entityNames: List[String], docs: List[ResourceDocPath]): List[String] = {
+    val docAmbiguities = docs.zipWithIndex.flatMap { case (doc, index) => resourceDocAmbiguities(doc, entityNames, docs.drop(index + 1)) }
+    val entityAmbiguities = entityNames.flatMap(entityNameAmbiguities(_, Nil))
+    (docAmbiguities ++ entityAmbiguities).distinct.sorted
   }
 
+  /** The names of the Dynamic Entities of a space (None for the system space), read from the database. */
+  def entityNamesIn(space: Option[String]): List[String] =
+    DynamicEntityProvider.connectorMethodProvider.vend.getDynamicEntities(space, false).map(_.entityName)
+
+  /** The Dynamic Resource Docs of a space (None for the system space), read from the database, not a cache. */
+  def resourceDocPathsIn(space: Option[String]): List[ResourceDocPath] =
+    DynamicResourceDocProvider.provider.vend.getAllInSpace(space)
+      .map(doc => ResourceDocPath(doc.dynamicResourceDocId, doc.requestVerb, doc.requestUrl, doc.partialFunctionName))
+
+  /** [[resourceDocAmbiguities]] for a doc about to be created (no id) or moved to a new verb or path (its id). */
+  def storedResourceDocAmbiguities(space: Option[String], dynamicResourceDocId: Option[String], verb: String, path: String, name: String): List[String] =
+    resourceDocAmbiguities(ResourceDocPath(dynamicResourceDocId, verb, path, name), entityNamesIn(space), resourceDocPathsIn(space))
+
+  /** [[entityNameAmbiguities]] for a Dynamic Entity about to be created or renamed in a space. */
+  def storedEntityNameAmbiguities(space: Option[String], entityName: String): List[String] =
+    entityNameAmbiguities(entityName, resourceDocPathsIn(space))
+
+  /** [[ambiguitiesInSpace]] for a space as the database holds it now. */
+  def storedAmbiguitiesInSpace(space: Option[String]): List[String] =
+    ambiguitiesInSpace(entityNamesIn(space), resourceDocPathsIn(space))
+
   /** A Dynamic Entity record response as a Domain API returns it: without `bank_id`. */
   def responseUnderDomainApi(response: JObject): JObject =
     JObject(response.obj.filterNot(_._1 == "bank_id"))

obp-api/src/main/scala/code/api/dynamic/domainapi/Http4sDomainApi.scala

@@ -53,9 +53,14 @@ import org.json4s.native.JsonMethods.compact
  * A call under a base path is rewritten to the OBP URL of the same endpoint (see [[DomainApiPaths]]) and
  * handed to the handler that serves that URL, so authentication, Roles, Consents, rate limiting,
  * row-level access, field restrictions, Dynamic Query checks and metrics all run exactly as they do for
- * the OBP URL. A Domain API grants nothing. A Dynamic Entity of the space is tried first, then a Dynamic
- * Resource Doc of the space; registering or updating a Domain API is refused while two of the space's
- * endpoints would answer the same verb and path, so the order only matters for a clash created later.
+ * the OBP URL. A Domain API adds no access of its own: a call under its base path is allowed or refused
+ * exactly as the same call to the OBP URL would be.
+ *
+ * A Dynamic Resource Doc of the space is tried first, then a Dynamic Entity of the space. Every space is
+ * kept free of ambiguous paths (see [[DomainApiPaths.resourceDocAmbiguities]]), with or without a Domain
+ * API, so at most one of them matches a call and the order does not change which one answers. Should an
+ * ambiguity that predates those rules remain, the Dynamic Resource Doc wins: its path names a literal
+ * segment where the Dynamic Entity's has a record id, so it is the more specific of the two.
  *
  * `BASE_PATH/openapi.json` and `BASE_PATH/openapi.yaml` serve the Domain API's own OpenAPI document.
  *
@@ -89,8 +94,8 @@ object Http4sDomainApi extends MdcLoggable {
             case _ =>
               val marked = req.withAttribute(domainApiCallKey,
                 DomainApiCall(route.domainApiId, route.basePath, req.uri.path.renderString))
-              Http4sDynamicEntity.wrappedRoutesDynamicEntityV700.run(withPath(marked, DomainApiPaths.dynamicEntityPath(route.bankId, rest)))
-                .orElse(Http4sDynamicEndpoint.wrappedRoutesDynamicEndpoint.run(withPath(marked, DomainApiPaths.dynamicResourceDocPath(route.bankId, rest))))
+              Http4sDynamicEndpoint.wrappedRoutesDynamicEndpoint.run(withPath(marked, DomainApiPaths.dynamicResourceDocPath(route.bankId, rest)))
+                .orElse(Http4sDynamicEntity.wrappedRoutesDynamicEntityV700.run(withPath(marked, DomainApiPaths.dynamicEntityPath(route.bankId, rest))))
           }
       }
     }

obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala

@@ -67,7 +67,7 @@ import scala.concurrent.Future
  *
  * Notes on the port:
  *   - The dynamic-entity set is runtime-mutable (`DynamicEntityHelper.definitionsMap` is
- *     re-queried per request), so this service does NOT use `ResourceDocMiddleware`
+ *     kept only until a definition changes), so this service does NOT use `ResourceDocMiddleware`
  *     (whose ResourceDoc index is built once at startup).  Auth / role / bank checks are
  *     performed inline, exactly as the Lift handlers did.
  *   - The before/after authenticate interceptors carry auth-type / query-param / header-key

obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala

@@ -198,7 +198,41 @@ object DynamicEntityHelper {
   private val implementedInApiVersion = ApiVersion.v4_0_0
 
   // Keyed by (bank id as published, entity name): SYS for the system space, never None or "".
-  def definitionsMap: Map[(String, String), DynamicEntityInfo] = NewStyle.function.getDynamicEntities(None, true).map(it => ((DynamicEntitySpace.bankIdOrSystem(it.bankId), it.entityName), DynamicEntityInfo(it.metadataJson, it.entityName, it.bankId, it.hasPersonalEntity, it.hasPublicAccess, it.hasCommunityAccess, it.personalRequiresRole, it.useRowLevelAccess, it.authMode))).toMap
+  //
+  // It is asked for many times per request (the access checks, each field's read check, the resource
+  // docs), and building it reads every definition and parses each one's JSON: with 120 definitions
+  // that was ~5 ms a time, and a Dynamic Query returning 11 records spent ~0.5 s rebuilding it. So
+  // the built map is kept for dynamicEntity.definitions_map.cache.ttl.seconds (default 300), and
+  // forgotten the moment a definition is created, updated or deleted on this node
+  // (MappedDynamicEntityProvider calls forgetDefinitions). Another node sees such a change when its
+  // copy expires. Not kept in test mode, where a test may change the table behind the provider.
+  private val definitionsMapTtlMillis: Long =
+    if (net.liftweb.util.Props.testMode) 0L
+    else APIUtil.getPropsAsLongValue("dynamicEntity.definitions_map.cache.ttl.seconds", 300L) * 1000L
+  // (built at, map). Each change replaces the token, and a build keeps its map only while the token it
+  // started under is still the current one, so a build that started before a change is not kept after it.
+  @volatile private var definitionsMapCache: Option[(Long, Map[(String, String), DynamicEntityInfo])] = None
+  @volatile private var definitionsToken: AnyRef = new Object
+
+  /** Forget the kept definitions map, so the next use reads the definitions again. */
+  def forgetDefinitions(): Unit = {
+    definitionsToken = new Object
+    definitionsMapCache = None
+  }
+
+  def definitionsMap: Map[(String, String), DynamicEntityInfo] = {
+    val now = System.currentTimeMillis()
+    definitionsMapCache match {
+      case Some((builtAt, map)) if now - builtAt < definitionsMapTtlMillis => map
+      case _ =>
+        val token = definitionsToken
+        val map = buildDefinitionsMap
+        if (definitionsMapTtlMillis > 0 && (definitionsToken eq token)) definitionsMapCache = Some((now, map))
+        map
+    }
+  }
+
+  private def buildDefinitionsMap: Map[(String, String), DynamicEntityInfo] = NewStyle.function.getDynamicEntities(None, true).map(it => ((DynamicEntitySpace.bankIdOrSystem(it.bankId), it.entityName), DynamicEntityInfo(it.metadataJson, it.entityName, it.bankId, it.hasPersonalEntity, it.hasPublicAccess, it.hasCommunityAccess, it.personalRequiresRole, it.useRowLevelAccess, it.authMode))).toMap
 
   /**
    * The definition of one entity in one space, or None when that space holds no such entity.
@@ -1355,12 +1389,19 @@ object DynamicEntityInfo {
   /**
    * Whether one caller may read each field, as `(entity, field) => Boolean`, for reads that span
    * several entities (joins, Dynamic Queries): [[mayReadField]], with each entity's public-access
-   * question answered once per entity.
+   * question answered once per entity, and each field's answer once per field.
+   *
+   * The answer doesn't depend on the record, and [[mayReadField]] looks the definition up through
+   * [[DynamicEntityHelper.definitionOf]], which rebuilds the map of every definition (and reads them
+   * all from the database when the definition cache is off, as in dev mode). A Dynamic Query asks
+   * once per field of every record it returns, so without remembering the answers a page of n
+   * records rebuilt that map n x fields times: about 0.5 s for 11 activities locally.
    */
   def fieldReader(bankId: Option[String], userIdOpt: Option[String], consumerId: String): (String, String) => Boolean = {
     val viaPublic = scala.collection.mutable.Map[String, Boolean]()
-    (entityName, fieldName) => mayReadField(bankId, entityName, fieldName, userIdOpt,
-      viaPublic.getOrElseUpdate(entityName, readsViaPublicAccess(bankId, entityName, userIdOpt, consumerId)))
+    val answers = scala.collection.mutable.Map[(String, String), Boolean]()
+    (entityName, fieldName) => answers.getOrElseUpdate((entityName, fieldName), mayReadField(bankId, entityName, fieldName, userIdOpt,
+      viaPublic.getOrElseUpdate(entityName, readsViaPublicAccess(bankId, entityName, userIdOpt, consumerId))))
   }
 
   def mayReadRecords(bankId: Option[String], entityName: String, userIdOpt: Option[String], consumerId: String): Boolean =

obp-api/src/main/scala/code/api/util/ErrorMessages.scala

@@ -117,7 +117,8 @@ object ErrorMessages {
   val InvalidDomainApiVersion = "OBP-09028: Invalid version. It must be a semantic version, MAJOR.MINOR.PATCH (for example 1.0.0), whose MAJOR equals the N of the vN that ends base_path."
   val DomainApiNotFound = "OBP-09029: Domain API not found in this space. Please specify a valid value for DOMAIN_API_ID."
   val InvalidDomainApiTitle = "OBP-09030: Invalid title or description. title must be 1 to 255 characters and description at most 2000."
-  val DomainApiPathClash = "OBP-09031: Two endpoints of this space would answer the same verb and path under a Domain API, so it cannot be published until one of them is renamed or removed: "
+  val DomainApiPathClash = "OBP-09031: Endpoints of this space are ambiguous with each other, so a Domain API cannot publish the space until one of them is renamed or removed: "
+  val DynamicPathAmbiguous = "OBP-09032: This would make a path ambiguous in its space. The Dynamic Entities and Dynamic Resource Docs of a space share one set of paths, which a Domain API can publish under its base path at any time, so a Dynamic Resource Doc's path may not start with a path variable, with my, public, community, openapi.json or openapi.yaml, or with the name of one of the space's Dynamic Entities; a Dynamic Entity may not be named my, public, community, openapi.json or openapi.yaml, nor after the first segment of one of the space's Dynamic Resource Docs; and two Dynamic Resource Docs of one verb may not both match one request. Rename one of them: "
 
 
   // General messages (OBP-10XXX)