diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index 5192975638..5dcd849c40 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -94,6 +94,10 @@ starConnector_supported_types=mapped,internal ## DynamicEntity cache time-to-live in seconds, default is 30, the value is 0 at test environment ## no 0 value will cause new dynamic entity will be shown after that seconds dynamicEntity.cache.ttl.seconds=30 +## How long each node keeps the parsed map of every Dynamic Entity definition, which the access and +## field checks consult many times per request. A definition created, updated or deleted on this node +## is seen at once; on another node, within this many seconds. Default 300. Not kept in test mode. +#dynamicEntity.definitions_map.cache.ttl.seconds=300 ## no 0 value will cause new dynamic endpoints will be shown after that seconds, default is 32, the value is 0 at test environment ## DynamicEndpoint cache time-to-live in seconds, default is 0, should set a no 0 value at product environment dynamicEndpoint.cache.ttl.seconds=32 diff --git a/obp-api/src/main/scala/code/api/dynamic/domainapi/DomainApiPaths.scala b/obp-api/src/main/scala/code/api/dynamic/domainapi/DomainApiPaths.scala index 6373fbfdb6..faa606a3a7 100644 --- a/obp-api/src/main/scala/code/api/dynamic/domainapi/DomainApiPaths.scala +++ b/obp-api/src/main/scala/code/api/dynamic/domainapi/DomainApiPaths.scala @@ -31,8 +31,10 @@ import cats.effect.IO import cats.effect.unsafe.implicits.global import code.api.Constant.ApiPathZero import code.api.berlin.group.ConstantsBG -import code.api.util.APIUtil.ResourceDoc +import code.api.util.APIUtil.ResourceDoc.isPathVariable import code.domainapi.DomainApiRoute +import code.dynamicEntity.DynamicEntityProvider +import code.dynamicResourceDoc.DynamicResourceDocProvider import com.openbankproject.commons.util.{ApiShortVersions, ApiStandards, ApiVersion} import org.json4s.JsonAST.{JObject, JValue} @@ -78,7 +80,7 @@ object DomainApiPaths { /** * Path segments a Dynamic Entity URL already gives a meaning to right after the space, and the names of * the Domain API's own documentation. A Dynamic Resource Doc whose path starts with one of them would be - * hidden under a Domain API, so it counts as a clash. + * hidden under a Domain API, so it is refused, and so is a Dynamic Entity named one of them. */ val reservedUnderBasePath: Set[String] = Set("my", "public", "community", "openapi.json", "openapi.yaml") @@ -145,30 +147,92 @@ object DomainApiPaths { } } - /** A path template with each placeholder (an all-capitals segment) reduced to one form, for comparison. */ - private def templateKey(path: String): String = - path.split("/").filter(_.nonEmpty).map(s => if (s.matches("[A-Z][A-Z0-9_]*")) "{}" else s).mkString("/", "/", "") + /** This is one Dynamic Resource Doc as the path rules below see it. */ + case class ResourceDocPath(dynamicResourceDocId: Option[String], verb: String, path: String, name: String) { + def segments: List[String] = path.split("/").filter(_.nonEmpty).toList + def describe: String = s"${verb.toUpperCase} $path ($name)" + } + + /** + * Two paths are ambiguous when one request could match both: they have as many segments, and at each + * position the segments are equal or at least one of them is a path variable (an all-capitals segment). + */ + def ambiguous(a: List[String], b: List[String]): Boolean = + a.length == b.length && a.zip(b).forall { case (x, y) => x == y || isPathVariable(x) || isPathVariable(y) } /** - * The verb and path pairs that more than one endpoint of the space would publish, and the Dynamic - * Resource Docs whose path starts with a segment a Dynamic Entity URL or the documentation already uses. - * Each is described for the person who has to resolve it. + * This lists why a Dynamic Resource Doc's path would be ambiguous in its space, given the space's + * Dynamic Entity names and its other Dynamic Resource Docs; empty when it is not. + * + * The Dynamic Entities and Dynamic Resource Docs of one space share one set of paths when a Domain API + * publishes the space under its base path, where a Dynamic Entity owns every path that starts with its + * name. The rules hold in every space, with or without a Domain API, so that one can be registered over + * any space at any time. A path may not start with a path variable (it would match every entity name), + * with a segment a Dynamic Entity URL or the Domain API's documentation already uses, or with the name + * of one of the space's Dynamic Entities, and no other doc of the same verb may match a request it + * matches. A doc is never compared with itself (the same dynamicResourceDocId). */ - def clashes(space: String, docs: List[ResourceDoc]): List[String] = { - val published = docs.flatMap(doc => publishedPath(space, doc.requestUrl).map(path => (doc, path))) - val duplicates = published - .groupBy { case (doc, path) => (doc.requestVerb.toUpperCase, templateKey(path)) } - .collect { case ((verb, key), entries) if entries.length > 1 => - s"$verb $key (${entries.map(_._1.partialFunctionName).sorted.mkString(", ")})" - }.toList - val hidden = published.collect { - case (doc, path) if doc.requestUrl.contains(s"/$dynamicResourceDocSegment/") && - reservedUnderBasePath.contains(path.split("/").filter(_.nonEmpty).headOption.getOrElse("")) => - s"${doc.requestVerb.toUpperCase} $path (${doc.partialFunctionName}) starts with a reserved segment" + def resourceDocAmbiguities(doc: ResourceDocPath, entityNames: List[String], otherDocs: List[ResourceDocPath]): List[String] = + doc.segments match { + case Nil => List(s"${doc.describe} has no path segment") + case first :: _ => + val variableFirst = + if (isPathVariable(first)) List(s"${doc.describe} starts with the path variable $first, which would also match every Dynamic Entity name") else Nil + val reserved = + if (reservedUnderBasePath.contains(first)) List(s"${doc.describe} starts with $first, which a Dynamic Entity URL or a Domain API's documentation already uses") else Nil + val entities = entityNames.filter(_.equalsIgnoreCase(first)) + .map(entityName => s"${doc.describe} starts with $first, the name of the Dynamic Entity $entityName") + val docs = otherDocs + .filterNot(other => doc.dynamicResourceDocId.isDefined && other.dynamicResourceDocId == doc.dynamicResourceDocId) + .filter(other => other.verb.equalsIgnoreCase(doc.verb) && ambiguous(other.segments, doc.segments)) + .map(other => s"${doc.describe} and ${other.describe} would both match one request") + variableFirst ++ reserved ++ entities ++ docs } - (duplicates ++ hidden).sorted + + /** + * This lists why a Dynamic Entity name would be ambiguous in its space, given the space's Dynamic + * Resource Docs: a doc whose path starts with the name (compared ignoring case), or a name that is a + * segment a Dynamic Entity URL or a Domain API's documentation already uses. Empty when it is neither. + */ + def entityNameAmbiguities(entityName: String, docs: List[ResourceDocPath]): List[String] = { + val reserved = + if (reservedUnderBasePath.contains(entityName.toLowerCase)) List(s"the Dynamic Entity name $entityName is a segment a Dynamic Entity URL or a Domain API's documentation already uses") else Nil + reserved ++ docs.filter(_.segments.headOption.exists(_.equalsIgnoreCase(entityName))) + .map(doc => s"${doc.describe} starts with $entityName, the name of the Dynamic Entity") + } + + /** + * This lists every ambiguity among a space's Dynamic Entities and Dynamic Resource Docs, each pair once. + * Writes are checked one at a time, so this finds only what predates the rules; a Domain API is + * refused over a space while the list is not empty. + */ + def ambiguitiesInSpace(entityNames: List[String], docs: List[ResourceDocPath]): List[String] = { + val docAmbiguities = docs.zipWithIndex.flatMap { case (doc, index) => resourceDocAmbiguities(doc, entityNames, docs.drop(index + 1)) } + val entityAmbiguities = entityNames.flatMap(entityNameAmbiguities(_, Nil)) + (docAmbiguities ++ entityAmbiguities).distinct.sorted } + /** The names of the Dynamic Entities of a space (None for the system space), read from the database. */ + def entityNamesIn(space: Option[String]): List[String] = + DynamicEntityProvider.connectorMethodProvider.vend.getDynamicEntities(space, false).map(_.entityName) + + /** The Dynamic Resource Docs of a space (None for the system space), read from the database, not a cache. */ + def resourceDocPathsIn(space: Option[String]): List[ResourceDocPath] = + DynamicResourceDocProvider.provider.vend.getAllInSpace(space) + .map(doc => ResourceDocPath(doc.dynamicResourceDocId, doc.requestVerb, doc.requestUrl, doc.partialFunctionName)) + + /** [[resourceDocAmbiguities]] for a doc about to be created (no id) or moved to a new verb or path (its id). */ + def storedResourceDocAmbiguities(space: Option[String], dynamicResourceDocId: Option[String], verb: String, path: String, name: String): List[String] = + resourceDocAmbiguities(ResourceDocPath(dynamicResourceDocId, verb, path, name), entityNamesIn(space), resourceDocPathsIn(space)) + + /** [[entityNameAmbiguities]] for a Dynamic Entity about to be created or renamed in a space. */ + def storedEntityNameAmbiguities(space: Option[String], entityName: String): List[String] = + entityNameAmbiguities(entityName, resourceDocPathsIn(space)) + + /** [[ambiguitiesInSpace]] for a space as the database holds it now. */ + def storedAmbiguitiesInSpace(space: Option[String]): List[String] = + ambiguitiesInSpace(entityNamesIn(space), resourceDocPathsIn(space)) + /** A Dynamic Entity record response as a Domain API returns it: without `bank_id`. */ def responseUnderDomainApi(response: JObject): JObject = JObject(response.obj.filterNot(_._1 == "bank_id")) diff --git a/obp-api/src/main/scala/code/api/dynamic/domainapi/Http4sDomainApi.scala b/obp-api/src/main/scala/code/api/dynamic/domainapi/Http4sDomainApi.scala index 7e2abaa898..c71d1919f4 100644 --- a/obp-api/src/main/scala/code/api/dynamic/domainapi/Http4sDomainApi.scala +++ b/obp-api/src/main/scala/code/api/dynamic/domainapi/Http4sDomainApi.scala @@ -53,9 +53,14 @@ import org.json4s.native.JsonMethods.compact * A call under a base path is rewritten to the OBP URL of the same endpoint (see [[DomainApiPaths]]) and * handed to the handler that serves that URL, so authentication, Roles, Consents, rate limiting, * row-level access, field restrictions, Dynamic Query checks and metrics all run exactly as they do for - * the OBP URL. A Domain API grants nothing. A Dynamic Entity of the space is tried first, then a Dynamic - * Resource Doc of the space; registering or updating a Domain API is refused while two of the space's - * endpoints would answer the same verb and path, so the order only matters for a clash created later. + * the OBP URL. A Domain API adds no access of its own: a call under its base path is allowed or refused + * exactly as the same call to the OBP URL would be. + * + * A Dynamic Resource Doc of the space is tried first, then a Dynamic Entity of the space. Every space is + * kept free of ambiguous paths (see [[DomainApiPaths.resourceDocAmbiguities]]), with or without a Domain + * API, so at most one of them matches a call and the order does not change which one answers. Should an + * ambiguity that predates those rules remain, the Dynamic Resource Doc wins: its path names a literal + * segment where the Dynamic Entity's has a record id, so it is the more specific of the two. * * `BASE_PATH/openapi.json` and `BASE_PATH/openapi.yaml` serve the Domain API's own OpenAPI document. * @@ -89,8 +94,8 @@ object Http4sDomainApi extends MdcLoggable { case _ => val marked = req.withAttribute(domainApiCallKey, DomainApiCall(route.domainApiId, route.basePath, req.uri.path.renderString)) - Http4sDynamicEntity.wrappedRoutesDynamicEntityV700.run(withPath(marked, DomainApiPaths.dynamicEntityPath(route.bankId, rest))) - .orElse(Http4sDynamicEndpoint.wrappedRoutesDynamicEndpoint.run(withPath(marked, DomainApiPaths.dynamicResourceDocPath(route.bankId, rest)))) + Http4sDynamicEndpoint.wrappedRoutesDynamicEndpoint.run(withPath(marked, DomainApiPaths.dynamicResourceDocPath(route.bankId, rest))) + .orElse(Http4sDynamicEntity.wrappedRoutesDynamicEntityV700.run(withPath(marked, DomainApiPaths.dynamicEntityPath(route.bankId, rest)))) } } } diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala b/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala index 80946b2ec8..a3a48f7723 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala @@ -67,7 +67,7 @@ import scala.concurrent.Future * * Notes on the port: * - The dynamic-entity set is runtime-mutable (`DynamicEntityHelper.definitionsMap` is - * re-queried per request), so this service does NOT use `ResourceDocMiddleware` + * kept only until a definition changes), so this service does NOT use `ResourceDocMiddleware` * (whose ResourceDoc index is built once at startup). Auth / role / bank checks are * performed inline, exactly as the Lift handlers did. * - The before/after authenticate interceptors carry auth-type / query-param / header-key diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala index 1de39d0c0c..a130e2a7f1 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala @@ -198,7 +198,41 @@ object DynamicEntityHelper { private val implementedInApiVersion = ApiVersion.v4_0_0 // Keyed by (bank id as published, entity name): SYS for the system space, never None or "". - def definitionsMap: Map[(String, String), DynamicEntityInfo] = NewStyle.function.getDynamicEntities(None, true).map(it => ((DynamicEntitySpace.bankIdOrSystem(it.bankId), it.entityName), DynamicEntityInfo(it.metadataJson, it.entityName, it.bankId, it.hasPersonalEntity, it.hasPublicAccess, it.hasCommunityAccess, it.personalRequiresRole, it.useRowLevelAccess, it.authMode))).toMap + // + // It is asked for many times per request (the access checks, each field's read check, the resource + // docs), and building it reads every definition and parses each one's JSON: with 120 definitions + // that was ~5 ms a time, and a Dynamic Query returning 11 records spent ~0.5 s rebuilding it. So + // the built map is kept for dynamicEntity.definitions_map.cache.ttl.seconds (default 300), and + // forgotten the moment a definition is created, updated or deleted on this node + // (MappedDynamicEntityProvider calls forgetDefinitions). Another node sees such a change when its + // copy expires. Not kept in test mode, where a test may change the table behind the provider. + private val definitionsMapTtlMillis: Long = + if (net.liftweb.util.Props.testMode) 0L + else APIUtil.getPropsAsLongValue("dynamicEntity.definitions_map.cache.ttl.seconds", 300L) * 1000L + // (built at, map). Each change replaces the token, and a build keeps its map only while the token it + // started under is still the current one, so a build that started before a change is not kept after it. + @volatile private var definitionsMapCache: Option[(Long, Map[(String, String), DynamicEntityInfo])] = None + @volatile private var definitionsToken: AnyRef = new Object + + /** Forget the kept definitions map, so the next use reads the definitions again. */ + def forgetDefinitions(): Unit = { + definitionsToken = new Object + definitionsMapCache = None + } + + def definitionsMap: Map[(String, String), DynamicEntityInfo] = { + val now = System.currentTimeMillis() + definitionsMapCache match { + case Some((builtAt, map)) if now - builtAt < definitionsMapTtlMillis => map + case _ => + val token = definitionsToken + val map = buildDefinitionsMap + if (definitionsMapTtlMillis > 0 && (definitionsToken eq token)) definitionsMapCache = Some((now, map)) + map + } + } + + private def buildDefinitionsMap: Map[(String, String), DynamicEntityInfo] = NewStyle.function.getDynamicEntities(None, true).map(it => ((DynamicEntitySpace.bankIdOrSystem(it.bankId), it.entityName), DynamicEntityInfo(it.metadataJson, it.entityName, it.bankId, it.hasPersonalEntity, it.hasPublicAccess, it.hasCommunityAccess, it.personalRequiresRole, it.useRowLevelAccess, it.authMode))).toMap /** * The definition of one entity in one space, or None when that space holds no such entity. @@ -1355,12 +1389,19 @@ object DynamicEntityInfo { /** * Whether one caller may read each field, as `(entity, field) => Boolean`, for reads that span * several entities (joins, Dynamic Queries): [[mayReadField]], with each entity's public-access - * question answered once per entity. + * question answered once per entity, and each field's answer once per field. + * + * The answer doesn't depend on the record, and [[mayReadField]] looks the definition up through + * [[DynamicEntityHelper.definitionOf]], which rebuilds the map of every definition (and reads them + * all from the database when the definition cache is off, as in dev mode). A Dynamic Query asks + * once per field of every record it returns, so without remembering the answers a page of n + * records rebuilt that map n x fields times: about 0.5 s for 11 activities locally. */ def fieldReader(bankId: Option[String], userIdOpt: Option[String], consumerId: String): (String, String) => Boolean = { val viaPublic = scala.collection.mutable.Map[String, Boolean]() - (entityName, fieldName) => mayReadField(bankId, entityName, fieldName, userIdOpt, - viaPublic.getOrElseUpdate(entityName, readsViaPublicAccess(bankId, entityName, userIdOpt, consumerId))) + val answers = scala.collection.mutable.Map[(String, String), Boolean]() + (entityName, fieldName) => answers.getOrElseUpdate((entityName, fieldName), mayReadField(bankId, entityName, fieldName, userIdOpt, + viaPublic.getOrElseUpdate(entityName, readsViaPublicAccess(bankId, entityName, userIdOpt, consumerId)))) } def mayReadRecords(bankId: Option[String], entityName: String, userIdOpt: Option[String], consumerId: String): Boolean = diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index 11e853287e..206515e69d 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -117,7 +117,8 @@ object ErrorMessages { val InvalidDomainApiVersion = "OBP-09028: Invalid version. It must be a semantic version, MAJOR.MINOR.PATCH (for example 1.0.0), whose MAJOR equals the N of the vN that ends base_path." val DomainApiNotFound = "OBP-09029: Domain API not found in this space. Please specify a valid value for DOMAIN_API_ID." val InvalidDomainApiTitle = "OBP-09030: Invalid title or description. title must be 1 to 255 characters and description at most 2000." - val DomainApiPathClash = "OBP-09031: Two endpoints of this space would answer the same verb and path under a Domain API, so it cannot be published until one of them is renamed or removed: " + val DomainApiPathClash = "OBP-09031: Endpoints of this space are ambiguous with each other, so a Domain API cannot publish the space until one of them is renamed or removed: " + val DynamicPathAmbiguous = "OBP-09032: This would make a path ambiguous in its space. The Dynamic Entities and Dynamic Resource Docs of a space share one set of paths, which a Domain API can publish under its base path at any time, so a Dynamic Resource Doc's path may not start with a path variable, with my, public, community, openapi.json or openapi.yaml, or with the name of one of the space's Dynamic Entities; a Dynamic Entity may not be named my, public, community, openapi.json or openapi.yaml, nor after the first segment of one of the space's Dynamic Resource Docs; and two Dynamic Resource Docs of one verb may not both match one request. Rename one of them: " // General messages (OBP-10XXX) diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 8314c265e9..27ed25da3e 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -4264,11 +4264,11 @@ object Glossary extends MdcLoggable { | |Dynamic Entity names are not renamed: only the part of the URL before them is. The endpoints that create and change definitions stay at their OBP URLs; a Domain API publishes the endpoints that serve and take data. Dynamic Endpoints made from a Swagger file are not published under a Domain API yet. | -|**It only renames.** A call under a base path is rewritten to the OBP URL and runs exactly as a call to that URL would: the same authentication, Roles, Consents, rate limits, row-level access and field restrictions, and the same API Metrics. A Domain API grants nothing. The one difference in a response is that a Dynamic Entity record response leaves out `bank_id`, because the base path already fixes the space; the OpenAPI document's examples leave it out too. +|**It only renames.** A call under a base path is rewritten to the OBP URL and runs exactly as a call to that URL would: the same authentication, Roles, Consents, rate limits, row-level access and field restrictions, and the same API Metrics. A Domain API gives no access the OBP URL would not give. The one difference in a response is that a Dynamic Entity record response leaves out `bank_id`, because the base path already fixes the space; the OpenAPI document's examples leave it out too. | |**Base path and version.** The base path is two to five segments of lowercase letters, digits, hyphens and dots, ending with the major version as `vN`, for example `carbon-registry/v1`. It may not start with a segment OBP serves itself (such as `obp` or `open-banking`), and may not overlap another Domain API's base path. The Domain API's `version` is its full semantic version, MAJOR.MINOR.PATCH, whose MAJOR is the N of the base path. A compatible change, such as a new optional field or a new Dynamic Query, edits `version` (or leaves it) and keeps every URL; a breaking change gets a new Domain API with a new base path, for example `carbon-registry/v2`, which can run alongside the old one while clients move. OBP never changes the version itself. For Dynamic Entities OBP already keeps changes compatible once an entity holds records (only optional properties may be added); a Dynamic Resource Doc can be changed in any way, so keeping its changes compatible is up to its author. | -|**Clashes.** Under a base path the Dynamic Entities and Dynamic Resource Docs of the space share one set of paths. A Domain API is refused (${ErrorMessages.DomainApiPathClash.takeWhile(_ != ':')}) while two of them would answer the same verb and path, or while a Dynamic Resource Doc's path starts with `my`, `public`, `community`, `openapi.json` or `openapi.yaml`. If a clash is created later, the Dynamic Entity answers. +|**No ambiguous paths.** Under a base path the Dynamic Entities and Dynamic Resource Docs of the space share one set of paths, and a Dynamic Entity owns every path that starts with its name. Because a Domain API may be registered over any space at any time, OBP keeps every space unambiguous, with or without a Domain API: creating or changing a Dynamic Resource Doc is refused (${ErrorMessages.DynamicPathAmbiguous.takeWhile(_ != ':')}) when its path would start with a path variable, with `my`, `public`, `community`, `openapi.json` or `openapi.yaml`, or with the name of one of the space's Dynamic Entities, or when another doc of the same verb would match a request it matches (`/registry/REGISTRY_ID` and `/registry/summary` both match `/registry/summary`). Creating or renaming a Dynamic Entity is refused when one of the space's Dynamic Resource Docs starts with its name, or when it would be named one of those reserved segments. A space holding an ambiguity from before these rules can't have a Domain API (${ErrorMessages.DomainApiPathClash.takeWhile(_ != ':')}) until it is resolved. Under a base path a Dynamic Resource Doc is tried first, then a Dynamic Entity; with the rules above, at most one of them can answer a request. | |**Managing.** `/obp/v7.0.0/management/banks/BANK_ID/domain-apis` creates and lists a space's Domain APIs, and `/obp/v7.0.0/management/banks/BANK_ID/domain-apis/DOMAIN_API_ID` reads, updates and deletes one. BANK_ID is a bank's id or `SYS`. The Roles are CanCreateDomainApi, CanGetDomainApis, CanUpdateDomainApi and CanDeleteDomainApi, held at that BANK_ID. | diff --git a/obp-api/src/main/scala/code/api/util/NewStyle.scala b/obp-api/src/main/scala/code/api/util/NewStyle.scala index 21442b30d5..5a4386b548 100644 --- a/obp-api/src/main/scala/code/api/util/NewStyle.scala +++ b/obp-api/src/main/scala/code/api/util/NewStyle.scala @@ -33,6 +33,7 @@ import code.DynamicEndpoint.{DynamicEndpointProvider, DynamicEndpointT} import code.api.Constant.{SYSTEM_READ_ACCOUNTS_BERLIN_GROUP_VIEW_ID, SYSTEM_READ_BALANCES_BERLIN_GROUP_VIEW_ID} // checkPaymentServerTypeError was inlined from the retired BG v1.3 PIS builder (see PaymentInitiationServicePISApi.scala) import code.api.cache.{Caching, Redis} +import code.api.dynamic.domainapi.DomainApiPaths import code.api.dynamic.endpoint.helper.DynamicEndpointHelper import code.api.dynamic.entity.helper.{DynamicEntityHelper, DynamicEntityInfo} import code.api.util.APIUtil._ @@ -3555,6 +3556,11 @@ object NewStyle extends MdcLoggable{ return Helper.booleanToFuture(errorMsg, cc=callContext)(existsDynamicEntity.isEmpty).map(_.asInstanceOf[Box[DynamicEntityT]]) } + val ambiguities = DomainApiPaths.storedEntityNameAmbiguities(dynamicEntity.bankId, dynamicEntity.entityName) + if (ambiguities.nonEmpty) { + return Helper.booleanToFuture(s"$DynamicPathAmbiguous${ambiguities.mkString("; ")}", 409, cc=callContext)(false).map(_.asInstanceOf[Box[DynamicEntityT]]) + } + Future { val result = DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity) if (result.isDefined) invalidateDynamicResourceDocCaches() @@ -3584,6 +3590,13 @@ object NewStyle extends MdcLoggable{ return Helper.booleanToFuture(errorMsg, cc=callContext)(existsDynamicEntity.isEmpty).map(_.asInstanceOf[Box[DynamicEntityT]]) } + + // Only a new name is checked, so an entity caught in an ambiguity that predates the rule can + // still have its definition changed. + val ambiguities = DomainApiPaths.storedEntityNameAmbiguities(dynamicEntity.bankId, dynamicEntity.entityName) + if (ambiguities.nonEmpty) { + return Helper.booleanToFuture(s"$DynamicPathAmbiguous${ambiguities.mkString("; ")}", 409, cc=callContext)(false).map(_.asInstanceOf[Box[DynamicEntityT]]) + } } Future { diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index 0d7c41e55a..e72f62a5c8 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -53,6 +53,7 @@ import code.bankconnectors.LocalMappedConnectorInternal._ import code.consent.ConsentStatus import com.openbankproject.commons.model.enums.{AttributeCategory, AttributeType, UserInvitationPurpose} import java.util.Date +import code.api.dynamic.domainapi.DomainApiPaths import code.api.dynamic.endpoint.helper.DynamicEndpointHelper import code.api.dynamic.entity.helper.{DynamicEntityInfo, DynamicEntitySpace} import code.api.util.{ApiRole => ApiRoleObj} @@ -1762,7 +1763,7 @@ object Http4s400 { |${userAuthenticationMessage(true)}""", dynamicEntityRequestBodyExample.copy(bankId = None), dynamicEntityResponseBodyExample, - List(AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, UnknownError), + List(AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicPathAmbiguous, UnknownError), List(apiTagManageDynamicEntity, apiTagApi), Some(List(canCreateDynamicEntityDefinition)), http4sPartialFunction = Some(createSystemDynamicEntity)) @@ -1799,7 +1800,7 @@ object Http4s400 { |${userAuthenticationMessage(true)}""", dynamicEntityRequestBodyExample.copy(bankId = None), dynamicEntityResponseBodyExample, - List(BankNotFound, AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, UnknownError), + List(BankNotFound, AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicPathAmbiguous, UnknownError), List(apiTagManageDynamicEntity, apiTagApi), Some(List(canCreateDynamicEntityDefinition)), http4sPartialFunction = Some(createBankLevelDynamicEntity)) @@ -1836,7 +1837,7 @@ object Http4s400 { |${userAuthenticationMessage(true)}""", dynamicEntityRequestBodyExample.copy(bankId = None), dynamicEntityResponseBodyExample, - List(AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicEntityUpdateNotSchemaCompatible, UnknownError), + List(AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicEntityUpdateNotSchemaCompatible, DynamicPathAmbiguous, UnknownError), List(apiTagManageDynamicEntity, apiTagApi), Some(List(canUpdateDynamicEntityDefinition)), http4sPartialFunction = Some(updateSystemDynamicEntity)) @@ -1873,7 +1874,7 @@ object Http4s400 { |${userAuthenticationMessage(true)}""", dynamicEntityRequestBodyExample.copy(bankId = None), dynamicEntityResponseBodyExample, - List(BankNotFound, AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicEntityUpdateNotSchemaCompatible, UnknownError), + List(BankNotFound, AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicEntityUpdateNotSchemaCompatible, DynamicPathAmbiguous, UnknownError), List(apiTagManageDynamicEntity, apiTagApi), Some(List(canUpdateDynamicEntityDefinition)), http4sPartialFunction = Some(updateBankLevelDynamicEntity)) @@ -2001,7 +2002,7 @@ object Http4s400 { |${userAuthenticationMessage(true)}""", dynamicEntityRequestBodyExample.copy(bankId = None), dynamicEntityResponseBodyExample, - List(AuthenticatedUserIsRequired, InvalidMyDynamicEntityUser, InvalidJsonFormat, DynamicEntityUpdateNotSchemaCompatible, UnknownError), + List(AuthenticatedUserIsRequired, InvalidMyDynamicEntityUser, InvalidJsonFormat, DynamicEntityUpdateNotSchemaCompatible, DynamicPathAmbiguous, UnknownError), List(apiTagManageDynamicEntity, apiTagApi), None, http4sPartialFunction = Some(updateMyDynamicEntity)) @@ -9655,6 +9656,8 @@ object Http4s400 { _ <- code.util.Helper.booleanToFuture( s"$DynamicResourceDocAlreadyExists The combination of request_url(${body.requestUrl}) and request_verb(${body.requestVerb}) must be unique", cc = callContext) { !isExists } + ambiguities <- Future(DomainApiPaths.storedResourceDocAmbiguities(bankId, None, body.requestVerb, body.requestUrl, body.partialFunctionName)) + _ <- code.util.Helper.booleanToFuture(s"$DynamicPathAmbiguous${ambiguities.mkString("; ")}", 409, callContext) { ambiguities.isEmpty } result <- interceptOrApply(DYNAMIC_RESOURCE_DOC, ChangeOp.CREATE, None, 201, cc) { NewStyle.function.createJsonDynamicResourceDoc(bankId, body, callContext).map(_._1) } @@ -9674,7 +9677,13 @@ object Http4s400 { } _ <- validateDynamicResourceDocBody(body, cc) _ = compileDynamicResourceDoc(bankId.orElse(body.bankId), body, cc) - (_, callContext) <- NewStyle.function.getJsonDynamicResourceDocById(bankId, dynamicResourceDocId, Some(cc)) + (stored, callContext) <- NewStyle.function.getJsonDynamicResourceDocById(bankId, dynamicResourceDocId, Some(cc)) + // Only a new verb or path is checked, so a doc caught in an ambiguity that predates the rule can + // still have its body changed. + ambiguities <- Future( + if (stored.requestVerb == body.requestVerb && stored.requestUrl == body.requestUrl) Nil + else DomainApiPaths.storedResourceDocAmbiguities(bankId, Some(dynamicResourceDocId), body.requestVerb, body.requestUrl, body.partialFunctionName)) + _ <- code.util.Helper.booleanToFuture(s"$DynamicPathAmbiguous${ambiguities.mkString("; ")}", 409, callContext) { ambiguities.isEmpty } result <- interceptOrApply(DYNAMIC_RESOURCE_DOC, ChangeOp.UPDATE, Some(dynamicResourceDocId), 200, cc) { NewStyle.function.updateJsonDynamicResourceDoc( bankId, body.copy(dynamicResourceDocId = Some(dynamicResourceDocId)), callContext).map(_._1) @@ -9786,7 +9795,7 @@ object Http4s400 { |""", jsonDynamicResourceDoc.copy(dynamicResourceDocId = None), jsonDynamicResourceDoc, - List($AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, UnknownError), + List($AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicPathAmbiguous, UnknownError), List(apiTagDynamicResourceDoc), Some(List(canCreateDynamicResourceDoc)), http4sPartialFunction = Some(createDynamicResourceDoc) @@ -9804,7 +9813,7 @@ object Http4s400 { |""", jsonDynamicResourceDoc.copy(dynamicResourceDocId = None), jsonDynamicResourceDoc, - List($AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, UnknownError), + List($AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicPathAmbiguous, UnknownError), List(apiTagDynamicResourceDoc), Some(List(canUpdateDynamicResourceDoc)), http4sPartialFunction = Some(updateDynamicResourceDoc) @@ -9872,7 +9881,7 @@ object Http4s400 { |""", jsonDynamicResourceDoc.copy(dynamicResourceDocId = None), jsonDynamicResourceDoc, - List($BankNotFound, $AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, UnknownError), + List($BankNotFound, $AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicPathAmbiguous, UnknownError), List(apiTagDynamicResourceDoc), Some(List(canCreateBankLevelDynamicResourceDoc)), http4sPartialFunction = Some(createBankLevelDynamicResourceDoc) @@ -9890,7 +9899,7 @@ object Http4s400 { |""", jsonDynamicResourceDoc.copy(dynamicResourceDocId = None), jsonDynamicResourceDoc, - List($BankNotFound, $AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, UnknownError), + List($BankNotFound, $AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicPathAmbiguous, UnknownError), List(apiTagDynamicResourceDoc), Some(List(canUpdateBankLevelDynamicResourceDoc)), http4sPartialFunction = Some(updateBankLevelDynamicResourceDoc) diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 37d1603a76..c90e6eedf6 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -509,13 +509,19 @@ object Http4s600 { !NewStyle.function.getMethodRoutings(Some("dynamicEntityProcess")) .exists(_.parameters.exists(p => p.key == "entityName" && p.value == dynamicEntity.entityName)) + // A failure the checks already shaped, an OBP message or the encoded failure Helper.booleanToFuture + // raises (with its own code, such as 409 for an ambiguous path), passes through as it is; anything + // else is wrapped as a 400 InvalidJsonFormat below. + private def isOwnApiFailure(e: Throwable): Boolean = + Option(e.getMessage).map(_.trim).exists(message => message.startsWith("OBP-") || message.startsWith("{\"failMsg\"")) + private[api] def createDynamicEntityV600(cc: CallContext, dynamicEntity: DynamicEntityCommons) = for { _ <- Helper.booleanToFuture(RowLevelAccessRequiresLocalBacking, 400, cc = Some(cc)) { localBackingOkForRowLevel(dynamicEntity) } // Wrap the connector call so a thrown RuntimeException (bad schema, etc.) // becomes a 400 InvalidJsonFormat — matches v6 Lift's dispatch wrapper. Full(result) <- NewStyle.function.createOrUpdateDynamicEntity(dynamicEntity, Some(cc)) .recoverWith { - case e: Throwable if !Option(e.getMessage).exists(_.startsWith("OBP-")) => + case e: Throwable if !isOwnApiFailure(e) => val json = org.json4s.native.Serialization.write( code.api.APIFailureNewStyle(s"$InvalidJsonFormat ${e.getMessage}", 400, Some(cc).map(_.toLight)) )(org.json4s.DefaultFormats) @@ -553,7 +559,7 @@ object Http4s600 { _ <- NewStyle.function.getDynamicEntityById(dynamicEntity.bankId, dynamicEntity.dynamicEntityId.getOrElse(""), Some(cc)) Full(result) <- NewStyle.function.createOrUpdateDynamicEntity(dynamicEntity, Some(cc)) .recoverWith { - case e: Throwable if !Option(e.getMessage).exists(_.startsWith("OBP-")) => + case e: Throwable if !isOwnApiFailure(e) => val json = org.json4s.native.Serialization.write( code.api.APIFailureNewStyle(s"$InvalidJsonFormat ${e.getMessage}", 400, Some(cc).map(_.toLight)) )(org.json4s.DefaultFormats) @@ -7367,7 +7373,7 @@ object Http4s600 { personal_requires_role = false, schema = com.openbankproject.commons.util.JsonAliases.parse("""{"description": "User preferences", "required": ["theme"], "properties": {"theme": {"type": "string", "minLength": 1, "maxLength": 20, "example": "dark", "description": "The UI theme preference", "indexed": true}, "language": {"type": "string", "minLength": 2, "maxLength": 5, "example": "en", "description": "ISO language code"}, "internal_note": {"type": "string", "example": "set by a privileged service", "description": "Field-level write-restricted (write_role_required)", "write_role_required": true}, "audit_ref": {"type": "string", "example": "AUD-0001", "description": "Field-level write-restricted via an explicit, shareable role (write_role)", "write_role": "CanWriteCustomerPreferencesAudit"}, "ssn": {"type": "string", "example": "123-45-6789", "description": "Field-level read-restricted (read_role_required)", "read_role_required": true}, "risk_score": {"type": "string", "example": "low", "description": "Field-level read-restricted via an explicit, shareable role (read_role)", "read_role": "CanReadCustomerPreferencesRisk"}}}""").asInstanceOf[org.json4s.JsonAST.JObject] ), - List($AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, UnknownError), + List($AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicPathAmbiguous, UnknownError), apiTagManageDynamicEntity :: apiTagApi :: Nil, Some(canCreateDynamicEntityDefinition :: Nil), authMode = code.api.util.APIUtil.UserOrApplication, @@ -7445,6 +7451,7 @@ object Http4s600 { $AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, + DynamicPathAmbiguous, UnknownError ), apiTagManageDynamicEntity :: apiTagApi :: Nil, @@ -7511,7 +7518,7 @@ object Http4s600 { has_public_access = false, schema = com.openbankproject.commons.util.JsonAliases.parse("""{"description": "User preferences updated", "required": ["theme"], "properties": {"theme": {"type": "string", "minLength": 1, "maxLength": 20, "example": "dark", "description": "The UI theme preference", "indexed": true}, "language": {"type": "string", "minLength": 2, "maxLength": 5, "example": "en", "description": "ISO language code"}, "notifications_enabled": {"type": "boolean", "example": "true", "description": "Whether to send notifications"}}}""").asInstanceOf[org.json4s.JsonAST.JObject] ), - List($AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, UnknownError), + List($AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, DynamicPathAmbiguous, UnknownError), apiTagManageDynamicEntity :: apiTagApi :: Nil, Some(canUpdateDynamicEntityDefinition :: Nil), http4sPartialFunction = Some(updateSystemDynamicEntity) @@ -7580,6 +7587,7 @@ object Http4s600 { $AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, + DynamicPathAmbiguous, UnknownError ), apiTagManageDynamicEntity :: apiTagApi :: Nil, @@ -7648,6 +7656,7 @@ object Http4s600 { List( $AuthenticatedUserIsRequired, InvalidJsonFormat, + DynamicPathAmbiguous, UnknownError ), apiTagManageDynamicEntity :: apiTagApi :: Nil, diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700DomainApis.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700DomainApis.scala index b99e789085..0a3cf45bfb 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700DomainApis.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700DomainApis.scala @@ -28,7 +28,7 @@ package code.api.v7_0_0 import cats.effect.IO import code.api.Constant.ApiPathZero -import code.api.dynamic.domainapi.{DomainApiPaths, Http4sDomainApi} +import code.api.dynamic.domainapi.DomainApiPaths import code.api.dynamic.entity.helper.DynamicEntitySpace import code.api.util.APIUtil.{EmptyBody, ResourceDoc} import code.api.util.ApiRole._ @@ -83,8 +83,9 @@ object Http4s700DomainApis { /** * The checks a registration must pass, on create and on update (`domainApiId` is the one being updated, * which may keep its own base path): the base path's shape, the version, the title, that no other - * Domain API's base path overlaps it, and that no two of the space's endpoints would publish the same - * verb and path. + * Domain API's base path overlaps it, and that none of the space's endpoints is ambiguous with another + * (see DomainApiPaths.ambiguitiesInSpace). Dynamic Entity and Dynamic Resource Doc writes are refused when + * they would make one ambiguous, so this only finds what predates that rule. */ private def check(spaceId: String, body: PostDomainApiJsonV700, domainApiId: Option[String], cc: CallContext): Future[Checked] = { val basePath = Option(body.base_path).map(_.trim).getOrElse("") @@ -96,7 +97,7 @@ object Http4s700DomainApis { .filterNot(other => domainApiId.contains(other.domainApiId)) .filter(other => DomainApiPaths.overlap(other.basePath, basePath)) .map(other => s"${other.basePath} (bank_id ${other.bankId})") - lazy val clashes = DomainApiPaths.clashes(spaceId, Http4sDomainApi.spaceDocs(spaceId)) + lazy val ambiguities = DomainApiPaths.storedAmbiguitiesInSpace(DynamicEntitySpace.bankIdOrNoneForSystem(spaceId)) for { _ <- Helper.booleanToFuture(s"$InvalidDomainApiBasePath${DomainApiPaths.reservedFirstSegments.toList.sorted.mkString(", ")}. Current base_path is $basePath: ${basePathProblem.getOrElse("")}.", 400, Some(cc)) { basePathProblem.isEmpty @@ -110,8 +111,8 @@ object Http4s700DomainApis { _ <- Helper.booleanToFuture(s"$DomainApiBasePathAlreadyExists${overlapping.mkString(", ")}", 409, Some(cc)) { overlapping.isEmpty } - _ <- Helper.booleanToFuture(s"$DomainApiPathClash${clashes.mkString("; ")}", 409, Some(cc)) { - clashes.isEmpty + _ <- Helper.booleanToFuture(s"$DomainApiPathClash${ambiguities.mkString("; ")}", 409, Some(cc)) { + ambiguities.isEmpty } } yield Checked(basePath, version, title, description) } @@ -138,9 +139,12 @@ object Http4s700DomainApis { |version, MAJOR.MINOR.PATCH, whose MAJOR is the N of the base path: a compatible change edits `version` |and leaves every URL alone, and a breaking change gets a new Domain API with a new base path. | - |A Domain API is refused while two endpoints of its space would answer the same verb and path under it, - |or while a Dynamic Resource Doc's path starts with a segment the Dynamic Entity URLs or the - |documentation use (`my`, `public`, `community`, `openapi.json`, `openapi.yaml`). + |A Domain API is refused while two endpoints of its space are ambiguous with each other under it: a + |Dynamic Resource Doc whose path starts with a path variable, with the name of one of the space's + |Dynamic Entities or with a segment the Dynamic Entity URLs or the documentation use (`my`, `public`, + |`community`, `openapi.json`, `openapi.yaml`), or two docs of one verb that would match one request. + |Creating or changing a Dynamic Entity or Dynamic Resource Doc that would cause one is refused in + |every space, so this only happens to a space that held one before that rule. | |On this instance a change is seen at once by the node that made it, and by the others within |${DomainApiDbProvider.cacheTtlSeconds} seconds. diff --git a/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicEntityProvider.scala b/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicEntityProvider.scala index 1ef2d8fd42..ea34e82808 100644 --- a/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicEntityProvider.scala +++ b/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicEntityProvider.scala @@ -119,6 +119,7 @@ object MappedDynamicEntityProvider extends DynamicEntityProvider with CustomJson .UseRowLevelAccess(dynamicEntity.useRowLevelAccess) .AuthMode(DynamicEntityAuthMode.normalise(dynamicEntity.authMode)) .saveMe() + code.api.dynamic.entity.helper.DynamicEntityHelper.forgetDefinitions() // DE_indexing: provision/refresh the projection for this definition's indexed scalar fields. // Guarded by projectionEnabled (default off); best-effort (a failure leaves the definition saved // and queries reporting pending, not a broken create). Fields passed explicitly because the new @@ -148,7 +149,7 @@ object MappedDynamicEntityProvider extends DynamicEntityProvider with CustomJson override def delete(dynamicEntity: DynamicEntityT): Box[Boolean] = Box.tryo{ - dynamicEntity match { + try dynamicEntity match { case v: DynamicEntity => DynamicEntity.delete_!(v) // Anything that is not one of our own rows is matched by name, and a name identifies an entity // only within one space: two spaces may each hold one called country. Without the bank id this @@ -158,7 +159,7 @@ object MappedDynamicEntityProvider extends DynamicEntityProvider with CustomJson case v => DynamicEntity.bulkDelete_!!( By(DynamicEntity.BankId, storedBankId(v.bankId)), By(DynamicEntity.EntityName, v.entityName)) - } + } finally code.api.dynamic.entity.helper.DynamicEntityHelper.forgetDefinitions() } private[this] def getByDynamicEntityId(dynamicEntityId: String): Box[DynamicEntity] = DynamicEntity.find(By(DynamicEntity.DynamicEntityId, dynamicEntityId)) diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala index 412363e261..aa23d2fb8e 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala @@ -79,6 +79,12 @@ trait DynamicResourceDocProvider { def getAllAndConvert[T: Manifest](bankId: Option[String], transform: JsonDynamicResourceDoc => T): List[T] + /** + * The docs of one space (None for the system space), read from the database every time. Unlike + * [[getAll]], which is cached and treats None as every space, this is what a check before a write needs. + */ + def getAllInSpace(bankId: Option[String]): List[JsonDynamicResourceDoc] + def create(bankId: Option[String], entity: JsonDynamicResourceDoc, createdByUserId: Option[String]): Box[JsonDynamicResourceDoc] def update(bankId: Option[String], entity: JsonDynamicResourceDoc, updatedByUserId: Option[String]): Box[JsonDynamicResourceDoc] def deleteById(bankId: Option[String], dynamicResourceDocId: String): Box[Boolean] diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala index a2bc52dd3a..823709974f 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala @@ -94,6 +94,10 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { } } + override def getAllInSpace(bankId: Option[String]): List[JsonDynamicResourceDoc] = + DynamicResourceDoc.findAll(By(DynamicResourceDoc.BankId, storedBankId(bankId))) + .map(DynamicResourceDoc.getJsonDynamicResourceDoc) + override def create(bankId: Option[String], entity: JsonDynamicResourceDoc, createdByUserId: Option[String]): Box[JsonDynamicResourceDoc]= tryo { val requestBody = entity.exampleRequestBody.map(json.compactRender(_)).orNull diff --git a/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala b/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala index ea7dabb871..ba93a07e58 100644 --- a/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala +++ b/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala @@ -32,6 +32,7 @@ import java.util.Date import code.abacrule.{AbacRule, AbacRuleEngine, MappedAbacRuleProvider} import code.api.Constant +import code.api.dynamic.domainapi.DomainApiPaths import code.api.dynamic.endpoint.helper.CompiledObjects import code.api.util.APIUtil.{dynamicCodeHash, getPropsAsBoolValue, getPropsAsIntValue, getPropsValue, sha256Hex} import code.api.util.DynamicUtil.Validation @@ -364,6 +365,14 @@ object MakerChecker extends MdcLoggable { case CREATE | UPDATE => for { body <- parseAs[JsonDynamicResourceDoc](request.proposedPayload) + // Checked again here, as the space may have changed since the request was made. + _ <- { + val unchangedPath = operation == UPDATE && p.getById(bankId, request.targetId) + .exists(stored => stored.requestVerb == body.requestVerb && stored.requestUrl == body.requestUrl) + val ambiguities = if (unchangedPath) Nil else DomainApiPaths.storedResourceDocAmbiguities( + bankId, Some(request.targetId).filter(_ => operation == UPDATE), body.requestVerb, body.requestUrl, body.partialFunctionName) + boolBox(ambiguities.isEmpty, s"${ErrorMessages.DynamicPathAmbiguous}${ambiguities.mkString("; ")}") + } _ <- compileBox("dynamic resource doc") { val compiled = CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang, bankId) compiled.validateDependency() diff --git a/obp-api/src/main/scala/code/model/OAuth.scala b/obp-api/src/main/scala/code/model/OAuth.scala index a0ebb6e109..6bcf38ac45 100644 --- a/obp-api/src/main/scala/code/model/OAuth.scala +++ b/obp-api/src/main/scala/code/model/OAuth.scala @@ -897,6 +897,11 @@ class Token extends LongKeyedMapper[Token]{ } } object Token extends Token with LongKeyedMetaMapper[Token]{ + // Every authenticated request looks its token up by key (DirectLogin does twice), so without this + // index each request scanned the whole table: 2.8 ms a lookup at 23k tokens, growing with every login. + // Not unique, so a database that already holds a duplicate key still starts. + override def dbIndexes = Index(key) :: super.dbIndexes + def gernerateVerifier(key : String) : Box[String] = { Token.find(key) match { case Full(tkn) => Full(tkn.gernerateVerifier) diff --git a/obp-api/src/test/scala/code/api/dynamic/domainapi/DomainApiPathRulesSpec.scala b/obp-api/src/test/scala/code/api/dynamic/domainapi/DomainApiPathRulesSpec.scala new file mode 100644 index 0000000000..3a6508a8b6 --- /dev/null +++ b/obp-api/src/test/scala/code/api/dynamic/domainapi/DomainApiPathRulesSpec.scala @@ -0,0 +1,90 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.api.dynamic.domainapi + +import code.api.dynamic.domainapi.DomainApiPaths.ResourceDocPath +import org.scalatest.{FlatSpec, Matchers} + +/** + * This suite checks the rules that keep a space's paths unambiguous: the Dynamic Entities and Dynamic + * Resource Docs of one space share one set of paths when a Domain API publishes them, and a Dynamic Entity + * owns every path that starts with its name. Pure unit tests, with no server and no database. + */ +class DomainApiPathRulesSpec extends FlatSpec with Matchers { + + private def doc(path: String, verb: String = "GET", id: Option[String] = None, name: String = "doc") = + ResourceDocPath(id, verb, path, name) + + "ambiguous" should "match paths of the same length whose segments are equal or a path variable at each position" in { + DomainApiPaths.ambiguous(List("registry", "REGISTRY_ID"), List("registry", "summary")) shouldBe true + DomainApiPaths.ambiguous(List("registry", "REGISTRY_ID"), List("registry", "SITE_ID")) shouldBe true + DomainApiPaths.ambiguous(List("registry", "summary"), List("registry", "totals")) shouldBe false + DomainApiPaths.ambiguous(List("registry"), List("registry", "summary")) shouldBe false + } + + "resourceDocAmbiguities" should "accept a path that starts with a literal no entity or other doc uses" in { + DomainApiPaths.resourceDocAmbiguities(doc("/registry/summary"), List("activity"), List(doc("/registry/totals"))) shouldBe empty + } + + it should "refuse a path that starts with the name of one of the space's Dynamic Entities, ignoring case" in { + DomainApiPaths.resourceDocAmbiguities(doc("/customers/summary"), List("customers"), Nil) should have size 1 + DomainApiPaths.resourceDocAmbiguities(doc("/Customers/summary"), List("customers"), Nil) should have size 1 + } + + it should "refuse a path that starts with a path variable, a reserved segment, or has no segment" in { + DomainApiPaths.resourceDocAmbiguities(doc("/ITEM_ID/summary"), Nil, Nil) should have size 1 + List("my", "public", "community", "openapi.json", "openapi.yaml").foreach { reserved => + DomainApiPaths.resourceDocAmbiguities(doc(s"/$reserved/summary"), Nil, Nil) should have size 1 + } + DomainApiPaths.resourceDocAmbiguities(doc("/"), Nil, Nil) should have size 1 + } + + it should "refuse a path another doc of the same verb would also match, and only of the same verb" in { + DomainApiPaths.resourceDocAmbiguities(doc("/registry/summary"), Nil, List(doc("/registry/REGISTRY_ID"))) should have size 1 + DomainApiPaths.resourceDocAmbiguities(doc("/registry/SITE_ID"), Nil, List(doc("/registry/REGISTRY_ID"))) should have size 1 + DomainApiPaths.resourceDocAmbiguities(doc("/registry/summary", verb = "POST"), Nil, List(doc("/registry/REGISTRY_ID"))) shouldBe empty + } + + it should "not compare a doc with itself when it is moved to a new path" in { + val stored = doc("/registry/REGISTRY_ID", id = Some("doc-1")) + DomainApiPaths.resourceDocAmbiguities(doc("/registry/SITE_ID", id = Some("doc-1")), Nil, List(stored)) shouldBe empty + } + + "entityNameAmbiguities" should "refuse a name that a doc's path starts with, or a reserved segment" in { + DomainApiPaths.entityNameAmbiguities("customers", List(doc("/customers/summary"))) should have size 1 + DomainApiPaths.entityNameAmbiguities("customers", List(doc("/registry/customers"))) shouldBe empty + DomainApiPaths.entityNameAmbiguities("public", Nil) should have size 1 + } + + "ambiguitiesInSpace" should "list each ambiguity in a space once" in { + val ambiguities = DomainApiPaths.ambiguitiesInSpace(List("customers"), + List(doc("/customers/summary", name = "a"), doc("/registry/REGISTRY_ID", name = "b"), doc("/registry/summary", name = "c"))) + ambiguities should have size 2 + DomainApiPaths.ambiguitiesInSpace(List("customers"), List(doc("/registry/summary"))) shouldBe empty + } +} diff --git a/obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala index e959679be4..a9b9e22fee 100644 --- a/obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala +++ b/obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala @@ -31,7 +31,7 @@ import code.api.Constant.DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON import code.api.dynamic.domainapi.Http4sDomainApi import code.api.util.APIUtil.OAuth._ -import code.api.util.ApiRole.{canCreateDomainApi, canDeleteDomainApi, canGetDomainApis, canUpdateDomainApi} +import code.api.util.ApiRole.{canCreateBankLevelDynamicResourceDoc, canCreateDomainApi, canCreateDynamicEntityDefinition, canDeleteDomainApi, canGetDomainApis, canUpdateBankLevelDynamicResourceDoc, canUpdateDomainApi} import code.api.util.ErrorMessages._ import code.api.v6_0_0.V600ServerSetup import code.domainapi.DomainApis @@ -43,6 +43,7 @@ import com.openbankproject.commons.util.ApiVersion import org.json4s.JsonAST._ import org.json4s.JsonDSL._ import org.json4s.native.JsonMethods.{compact, render} +import org.json4s.native.Serialization.write import org.scalatest.Tag import java.net.URLEncoder @@ -106,6 +107,16 @@ class DomainApisTest extends V600ServerSetup { methodBody = URLEncoder.encode(s"""{ "from": "$entityName", "select": ["name"], "envelope": { "rows": "names" } }""", "UTF-8"), programmingLang = "Query"), Some(owner)).openOrThrowException(s"doc in $space") + private def dynamicResourceDocsAt(bankId: String) = baseRequest / "obp" / "v4.0.0" / "management" / "banks" / bankId / "dynamic-resource-docs" + + /** The body of a Dynamic Query at `url` in a bank, reading `entityName`, as the management endpoints take it. */ + private def queryDocBody(bankId: String, url: String, entityName: String, functionName: String): String = + write(SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, bankId = Some(bankId), roles = "", partialFunctionName = functionName, + requestVerb = "GET", requestUrl = url, exampleRequestBody = None, errorResponseBodies = "OBP-50000: Unknown Error.", + methodBody = URLEncoder.encode(s"""{ "from": "$entityName", "select": ["name"], "envelope": { "rows": "names" } }""", "UTF-8"), + programmingLang = "Query")) + feature("Managing Domain APIs") { scenario("create, read, update and delete a Domain API at SYS, with the Roles held at SYS", @@ -263,4 +274,99 @@ class DomainApisTest extends V600ServerSetup { message(refused) should include(s"/$clashing") } } + + feature("Keeping a space's paths unambiguous, with or without a Domain API") { + + scenario("a Dynamic Resource Doc whose path starts with a Dynamic Entity's name is refused, in a space with no Domain API") { + val bankId = testBankId1.value + val name = s"owned_$suffix" + entityWithRecord(Some(bankId), name, "owned") + grantAt(bankId, canCreateBankLevelDynamicResourceDoc) + + When("a Dynamic Query is created at //summary") + val refused = makePostRequest(dynamicResourceDocsAt(bankId).POST <@ (user1), queryDocBody(bankId, s"/$name/summary", name, s"ownedSummary$suffix")) + Then("it is refused, naming the entity") + withClue(refused.body) { refused.code should equal(409) } + message(refused) should include(DynamicPathAmbiguous) + message(refused) should include(s"the Dynamic Entity $name") + + When("the same query is created at a path of its own") + val accepted = makePostRequest(dynamicResourceDocsAt(bankId).POST <@ (user1), queryDocBody(bankId, s"/report_$suffix/summary", name, s"ownedReport$suffix")) + Then("it is accepted") + withClue(accepted.body) { accepted.code should equal(201) } + } + + scenario("two Dynamic Resource Docs of one verb that would match one request are refused") { + val bankId = testBankId1.value + val name = s"listed_$suffix" + entityWithRecord(Some(bankId), name, "listed") + grantAt(bankId, canCreateBankLevelDynamicResourceDoc) + grantAt(bankId, canUpdateBankLevelDynamicResourceDoc) + val first = makePostRequest(dynamicResourceDocsAt(bankId).POST <@ (user1), queryDocBody(bankId, s"/sites_$suffix/SITE_ID", name, s"siteById$suffix")) + withClue(first.body) { first.code should equal(201) } + + When("a second doc is created whose literal segment the first one's path variable also matches") + val refused = makePostRequest(dynamicResourceDocsAt(bankId).POST <@ (user1), queryDocBody(bankId, s"/sites_$suffix/summary", name, s"siteSummary$suffix")) + Then("it is refused, naming the first") + withClue(refused.body) { refused.code should equal(409) } + message(refused) should include(DynamicPathAmbiguous) + message(refused) should include(s"siteById$suffix") + + When("a doc is moved onto such a path by an update") + val other = makePostRequest(dynamicResourceDocsAt(bankId).POST <@ (user1), queryDocBody(bankId, s"/regions_$suffix/summary", name, s"regionSummary$suffix")) + withClue(other.body) { other.code should equal(201) } + val otherId = (other.body \ "dynamic_resource_doc_id").extract[String] + val moved = makePutRequest((dynamicResourceDocsAt(bankId) / otherId).PUT <@ (user1), queryDocBody(bankId, s"/sites_$suffix/summary", name, s"regionSummary$suffix")) + Then("it is refused too") + withClue(moved.body) { moved.code should equal(409) } + message(moved) should include(DynamicPathAmbiguous) + } + + scenario("a Dynamic Entity named after the first segment of a Dynamic Resource Doc's path is refused") { + val bankId = testBankId1.value + val name = s"source_$suffix" + val taken = s"taken_$suffix" + entityWithRecord(Some(bankId), name, "source") + grantAt(bankId, canCreateBankLevelDynamicResourceDoc) + grantAt(bankId, canCreateDynamicEntityDefinition) + val doc = makePostRequest(dynamicResourceDocsAt(bankId).POST <@ (user1), queryDocBody(bankId, s"/$taken/names", name, s"takenNames$suffix")) + withClue(doc.body) { doc.code should equal(201) } + + When(s"a Dynamic Entity named $taken is created") + val definition = compact(render(("entity_name" -> taken) ~ ("has_personal_entity" -> false) ~ + ("schema" -> (("description" -> "Taken.") ~ ("required" -> List("name")) ~ + ("properties" -> ("name" -> (("type" -> "string") ~ ("example" -> "a name")))))))) + val refused = makePostRequest((baseRequest / "obp" / "v6.0.0" / "management" / "banks" / bankId / "dynamic-entities").POST <@ (user1), definition) + Then("it is refused, naming the doc") + withClue(refused.body) { refused.code should equal(409) } + message(refused) should include(DynamicPathAmbiguous) + message(refused) should include(s"/$taken/names") + } + + scenario("under a Domain API a Dynamic Resource Doc is tried before a Dynamic Entity") { + val bankId = testBankId1.value + grantAllAt(bankId) + val basePath = s"precedence-$suffix/v1" + val created = makePostRequest(domainApis(bankId).POST <@ (user1), registration(basePath, "1.0.0")) + withClue(created.body) { created.code should equal(201) } + + Given("an ambiguity written straight to the database, as one that predates the rules would be") + val name = s"older_$suffix" + entityWithRecord(Some(bankId), name, "older record") + queryDoc(Some(bankId), s"/$name/summary", name, s"olderSummary$suffix") + + try { + When("the doc's path is called under the base path, which the entity would read as a record id") + val response = makeGetRequest((under(basePath) / name / "summary").GET <@ (user1)) + Then("the Dynamic Resource Doc answers") + withClue(response.body) { response.code should equal(200) } + valuesOf(response.body \ "names", "name") should contain("older record") + And("the entity still answers its own paths") + makeGetRequest((under(basePath) / name).GET <@ (user1)).code should equal(200) + } finally { + DynamicResourceDocProvider.provider.vend.getByVerbAndUrl(Some(bankId), "GET", s"/$name/summary") + .foreach(doc => DynamicResourceDocProvider.provider.vend.deleteById(Some(bankId), doc.dynamicResourceDocId.getOrElse(""))) + } + } + } } diff --git a/scripts/resource_doc_baseline/parity_allowlist.json b/scripts/resource_doc_baseline/parity_allowlist.json index da3cec4f15..49573fcf40 100644 --- a/scripts/resource_doc_baseline/parity_allowlist.json +++ b/scripts/resource_doc_baseline/parity_allowlist.json @@ -1698,6 +1698,78 @@ "reason": "Group memberships are recorded in the GroupMembership table (2026-09-29): the description says the membership is recorded even when every Role is skipped, and that removal keeps a Role another of the user's Groups still grants.", "lift_digest": "2ea4d8ed48487fd253cf1633a68deb7a0ce5b2a6b554ecfb64ee798f4887297c", "http4s_digest": "2766a28acb728f97343d2de181f5ddbe89ff62dfdf61d191453a1ce4b6cf602f" + }, + { + "version": "v4_0_0", + "endpoint": "createBankLevelDynamicResourceDoc", + "field": "errorResponseBodies", + "reason": "Lists DynamicPathAmbiguous (OBP-09032): creating or moving a Dynamic Resource Doc onto a path that would be ambiguous in its space is refused with 409, so a Domain API can publish any space.", + "lift_digest": "9cbf4cf58ff9d965002032d8c2c75def1b679635d77e9f3247ae9c82900cb9fc", + "http4s_digest": "51bd74398ede5b040dbe38d6c95e6c47469f36036cd90e906dba444d5c4726be" + }, + { + "version": "v4_0_0", + "endpoint": "createDynamicResourceDoc", + "field": "errorResponseBodies", + "reason": "Lists DynamicPathAmbiguous (OBP-09032): creating or moving a Dynamic Resource Doc onto a path that would be ambiguous in its space is refused with 409, so a Domain API can publish any space.", + "lift_digest": "160f3a99ccf321a6af0129ef59ea81b833b7ff8f636a6557c8fa22a69ef726fd", + "http4s_digest": "ac8b5ea3a96b50a33022bfe6226e57f996a2d313d7568eafa396273522140a6a" + }, + { + "version": "v4_0_0", + "endpoint": "updateBankLevelDynamicResourceDoc", + "field": "errorResponseBodies", + "reason": "Lists DynamicPathAmbiguous (OBP-09032): creating or moving a Dynamic Resource Doc onto a path that would be ambiguous in its space is refused with 409, so a Domain API can publish any space.", + "lift_digest": "9cbf4cf58ff9d965002032d8c2c75def1b679635d77e9f3247ae9c82900cb9fc", + "http4s_digest": "51bd74398ede5b040dbe38d6c95e6c47469f36036cd90e906dba444d5c4726be" + }, + { + "version": "v4_0_0", + "endpoint": "updateDynamicResourceDoc", + "field": "errorResponseBodies", + "reason": "Lists DynamicPathAmbiguous (OBP-09032): creating or moving a Dynamic Resource Doc onto a path that would be ambiguous in its space is refused with 409, so a Domain API can publish any space.", + "lift_digest": "160f3a99ccf321a6af0129ef59ea81b833b7ff8f636a6557c8fa22a69ef726fd", + "http4s_digest": "ac8b5ea3a96b50a33022bfe6226e57f996a2d313d7568eafa396273522140a6a" + }, + { + "version": "v6_0_0", + "endpoint": "createBankLevelDynamicEntity", + "field": "errorResponseBodies", + "reason": "Lists DynamicPathAmbiguous (OBP-09032): creating or renaming a Dynamic Entity after the first segment of a Dynamic Resource Doc of its space, or after a reserved segment, is refused with 409, so a Domain API can publish any space.", + "lift_digest": "9cbf4cf58ff9d965002032d8c2c75def1b679635d77e9f3247ae9c82900cb9fc", + "http4s_digest": "51bd74398ede5b040dbe38d6c95e6c47469f36036cd90e906dba444d5c4726be" + }, + { + "version": "v6_0_0", + "endpoint": "createSystemDynamicEntity", + "field": "errorResponseBodies", + "reason": "Lists DynamicPathAmbiguous (OBP-09032): creating or renaming a Dynamic Entity after the first segment of a Dynamic Resource Doc of its space, or after a reserved segment, is refused with 409, so a Domain API can publish any space.", + "lift_digest": "160f3a99ccf321a6af0129ef59ea81b833b7ff8f636a6557c8fa22a69ef726fd", + "http4s_digest": "ac8b5ea3a96b50a33022bfe6226e57f996a2d313d7568eafa396273522140a6a" + }, + { + "version": "v6_0_0", + "endpoint": "updateBankLevelDynamicEntity", + "field": "errorResponseBodies", + "reason": "Lists DynamicPathAmbiguous (OBP-09032): creating or renaming a Dynamic Entity after the first segment of a Dynamic Resource Doc of its space, or after a reserved segment, is refused with 409, so a Domain API can publish any space.", + "lift_digest": "9cbf4cf58ff9d965002032d8c2c75def1b679635d77e9f3247ae9c82900cb9fc", + "http4s_digest": "51bd74398ede5b040dbe38d6c95e6c47469f36036cd90e906dba444d5c4726be" + }, + { + "version": "v6_0_0", + "endpoint": "updateMyDynamicEntity", + "field": "errorResponseBodies", + "reason": "Lists DynamicPathAmbiguous (OBP-09032): creating or renaming a Dynamic Entity after the first segment of a Dynamic Resource Doc of its space, or after a reserved segment, is refused with 409, so a Domain API can publish any space.", + "lift_digest": "8b9e9294a47baff632975336cec3da4c29f3ca3b8398ec5875e9b5d1712f0140", + "http4s_digest": "728dbf8a2667f373ad58c3ecaed8aa2bf972b841211dbf0e07717cb26d358e4e" + }, + { + "version": "v6_0_0", + "endpoint": "updateSystemDynamicEntity", + "field": "errorResponseBodies", + "reason": "Lists DynamicPathAmbiguous (OBP-09032): creating or renaming a Dynamic Entity after the first segment of a Dynamic Resource Doc of its space, or after a reserved segment, is refused with 409, so a Domain API can publish any space.", + "lift_digest": "160f3a99ccf321a6af0129ef59ea81b833b7ff8f636a6557c8fa22a69ef726fd", + "http4s_digest": "ac8b5ea3a96b50a33022bfe6226e57f996a2d313d7568eafa396273522140a6a" } ] }