diff --git a/.gitignore b/.gitignore
index ee6a47a274..a85daa1209 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,7 +8,7 @@
.settings
.metals
.vscode
-.claude/settings.local.json
+.claude/
*.code-workspace
.zed
.cursor
diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala
index d72df6a3ec..0e8f7f8067 100644
--- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala
+++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala
@@ -27,7 +27,7 @@ TESOBE (http://www.tesobe.com/)
package code.api.ResourceDocs1_4_0
-import code.api.Constant.{GET_DYNAMIC_RESOURCE_DOCS_TTL, GET_STATIC_RESOURCE_DOCS_TTL, HostName, PARAM_LOCALE}
+import code.api.Constant.{DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID, GET_DYNAMIC_RESOURCE_DOCS_TTL, GET_STATIC_RESOURCE_DOCS_TTL, HostName, PARAM_LOCALE}
import code.api.OBPRestHelper
import code.api.cache.Caching
import code.api.util.APIUtil._
@@ -297,6 +297,54 @@ trait ResourceDocsAPIMethods extends MdcLoggable {
List(apiTagDocumentation, apiTagApi)
)
+ localResourceDocs += ResourceDoc(
+ implementedInApiVersion,
+ "getBankLevelDynamicResourceDocsOpenAPI31",
+ "GET",
+ "/banks/BANK_ID/resource-docs/API_VERSION/openapi",
+ "Get Bank Level Dynamic OpenAPI 3.1 documentation",
+ s"""Returns OpenAPI 3.1 documentation for the dynamic resources of one bank: its Dynamic Entities,
+ |Dynamic Endpoints and Dynamic Resource Docs.
+ |
+ |BANK_ID is the bank whose dynamic resources you want. Use ${DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID} for the
+ |system level dynamic resources, the ones that belong to no bank.
+ |
+ |API_VERSION is the version you want documentation about e.g. v7.0.0
+ |
+ |Static endpoints belong to no bank, so this document only ever contains dynamic resources. To
+ |document every bank's dynamic resources at once, use /resource-docs/API_VERSION/openapi?content=dynamic
+ |
+ |## Query Parameters
+ |
+ |**tags** - Filter by endpoint tags (comma-separated list). Empty values will return error OBP-10053
+ |
+ |**functions** - Filter by function names (comma-separated list). Empty values will return error OBP-10054
+ |
+ |**locale** - Language for localized documentation, e.g. ?locale=en_GB. Invalid locales will return error OBP-10041
+ |
+ |For YAML format, use the corresponding endpoint: /banks/BANK_ID/resource-docs/API_VERSION/openapi.yaml
+ |
+ |Note: Resource Docs are cached, TTL is ${GET_DYNAMIC_RESOURCE_DOCS_TTL} seconds
+ |
+ |## Examples
+ |
+ |${getObpApiRoot}/v7.0.0/banks/${DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID}/resource-docs/v7.0.0/openapi
+ |${getObpApiRoot}/v7.0.0/banks/BANK_ID/resource-docs/v7.0.0/openapi.yaml
+ |
+ """,
+ EmptyBody,
+ EmptyBody,
+ BankNotFound ::
+ InvalidApiVersionString ::
+ ApiVersionNotSupported ::
+ InvalidLocale ::
+ InvalidTagsParameter ::
+ InvalidFunctionsParameter ::
+ UnknownError :: Nil,
+ List(apiTagDocumentation, apiTagApi),
+ Some(List(canReadDynamicResourceDocsAtOneBank))
+ )
+
implicit val formats = CustomJsonFormats.rolesMappedToClassesFormats
// avoid repeat execute method getSpecialInstructions, here save the calculate results.
@@ -507,7 +555,7 @@ trait ResourceDocsAPIMethods extends MdcLoggable {
// (don't keep a stale value left over from a different aggregated request that may have
// overwritten this var on the shared ResourceDoc).
val dynamicDocs = allDynamicResourceDocs
- .filter(rd => if (bankId.isDefined) rd.createdByBankId == bankId else true)
+ .filter(rd => bankId.forall(space => APIUtil.dynamicResourceDocBelongsToSpace(rd, space)))
.map { it =>
it.specifiedUrl = if (it.partialFunctionName.startsWith("dynamicEntity")) Some(s"/${it.implementedInApiVersion.urlPrefix}/${ApiVersion.`dynamic-entity`}${it.requestUrl}") else Some(s"/${it.implementedInApiVersion.urlPrefix}/${ApiVersion.`dynamic-endpoint`}${it.requestUrl}")
it
@@ -1147,36 +1195,36 @@ trait ResourceDocsAPIMethods extends MdcLoggable {
- def convertResourceDocsToOpenAPI31YAMLAndSetCache(cacheKey: String, requestedApiVersionString: String, resourceDocsJson: List[JSONFactory1_4_0.ResourceDocJson]) : String = {
+ def convertResourceDocsToOpenAPI31YAMLAndSetCache(cacheKey: String, requestedApiVersionString: String, resourceDocsJson: List[JSONFactory1_4_0.ResourceDocJson], setCache: (String, String) => Unit = Caching.setStaticSwaggerDocCache) : String = {
logger.debug(s"Generating OpenAPI 3.1 YAML-convertResourceDocsToOpenAPI31YAMLAndSetCache requestedApiVersion is $requestedApiVersionString")
val hostname = HostName
val openApiDoc = code.api.ResourceDocs1_4_0.OpenAPI31JSONFactory.createOpenAPI31Json(resourceDocsJson, requestedApiVersionString, hostname)
val openApiJValue = code.api.ResourceDocs1_4_0.OpenAPI31JSONFactory.OpenAPI31JsonFormats.toJValue(openApiDoc)
val yamlString = YAMLUtils.jValueToYAMLSafe(openApiJValue, "# Error converting to YAML")
- Caching.setStaticSwaggerDocCache(cacheKey, yamlString)
+ setCache(cacheKey, yamlString)
yamlString
}
- def convertResourceDocsToOpenAPI31JvalueAndSetCache(cacheKey: String, requestedApiVersionString: String, resourceDocsJson: List[JSONFactory1_4_0.ResourceDocJson]) : JValue = {
+ def convertResourceDocsToOpenAPI31JvalueAndSetCache(cacheKey: String, requestedApiVersionString: String, resourceDocsJson: List[JSONFactory1_4_0.ResourceDocJson], setCache: (String, String) => Unit = Caching.setStaticSwaggerDocCache) : JValue = {
logger.debug(s"Generating OpenAPI 3.1-convertResourceDocsToOpenAPI31JvalueAndSetCache requestedApiVersion is $requestedApiVersionString")
val hostname = HostName
val openApiDoc = code.api.ResourceDocs1_4_0.OpenAPI31JSONFactory.createOpenAPI31Json(resourceDocsJson, requestedApiVersionString, hostname)
val openApiJValue = code.api.ResourceDocs1_4_0.OpenAPI31JSONFactory.OpenAPI31JsonFormats.toJValue(openApiDoc)
val jsonString = json.compactRender(openApiJValue)
- Caching.setStaticSwaggerDocCache(cacheKey, jsonString)
+ setCache(cacheKey, jsonString)
openApiJValue
}
- def convertResourceDocsToSwaggerJvalueAndSetCache(cacheKey: String, requestedApiVersionString: String, resourceDocsJson: List[JSONFactory1_4_0.ResourceDocJson]) : JValue = {
+ def convertResourceDocsToSwaggerJvalueAndSetCache(cacheKey: String, requestedApiVersionString: String, resourceDocsJson: List[JSONFactory1_4_0.ResourceDocJson], setCache: (String, String) => Unit = Caching.setStaticSwaggerDocCache) : JValue = {
logger.debug(s"Generating Swagger-getResourceDocsSwaggerAndSetCache requestedApiVersion is $requestedApiVersionString")
val swaggerDocJsonJValue = getResourceDocsSwagger(requestedApiVersionString, resourceDocsJson).head
val jsonString = json.compactRender(swaggerDocJsonJValue)
- Caching.setStaticSwaggerDocCache(cacheKey, jsonString)
+ setCache(cacheKey, jsonString)
swaggerDocJsonJValue
}
diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/Http4sDynamicEndpoint.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/Http4sDynamicEndpoint.scala
index 5bb72002da..a87c085227 100644
--- a/obp-api/src/main/scala/code/api/dynamic/endpoint/Http4sDynamicEndpoint.scala
+++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/Http4sDynamicEndpoint.scala
@@ -127,12 +127,17 @@ object Http4sDynamicEndpoint extends MdcLoggable {
case Some(jv) => Full(jv)
case None => Empty
}
+ // Every outcome records an API metric, like any other endpoint: the handler's response
+ // under the authenticated call context (so the row names the User), and a failure to
+ // authenticate or authorise under the context the request arrived with.
val io: IO[Response[IO]] = for {
authedCcOpt <- IO.fromFuture(IO(doc.authCheckIO(partPath, bodyJValue, cc)))
authedCc = authedCcOpt.getOrElse(cc)
resp <- doc.dynamicHttp4sFunction.get.apply(req)(authedCc)
- } yield resp
- io.handleErrorWith(err => ErrorResponseConverter.toHttp4sResponse(err, cc))
+ .handleErrorWith(err => ErrorResponseConverter.toHttp4sResponse(err, authedCc))
+ recorded <- EndpointHelpers.recordMetricFor(resp)(authedCc)
+ } yield recorded
+ io.handleErrorWith(err => ErrorResponseConverter.toHttp4sResponse(err, cc).flatMap(EndpointHelpers.recordMetricFor(_)(cc)))
}
}
}
diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala
index 0df1554632..4ebf1c7d19 100644
--- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala
+++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala
@@ -33,7 +33,8 @@ import code.api.dynamic.endpoint.helper.practise.{DynamicEndpointCodeGenerator,
import code.api.dynamic.endpoint.helper.practise.PractiseEndpointGroup
import code.api.util.DynamicUtil.{DynamicCodeBody, Validation}
import code.api.util.APIUtil.{BooleanBody, DoubleBody, EmptyBody, LongBody, Http4sEndpointIO, PrimaryDataBody, ResourceDoc, StringBody, getDisabledEndpointOperationIds}
-import code.api.util.{APIUtil, CallContext, DynamicUtil}
+import code.api.util.{APIUtil, CallContext, DynamicUtil, ErrorMessages}
+import code.api.JsonResponseException
import net.liftweb.common.{Box, Failure, Full}
import org.json4s.{JNothing, JValue}
import org.json4s.JsonAST.{JBool, JDouble, JInt, JString}
@@ -111,10 +112,23 @@ object CompiledObjects {
* validate and the dry-run compile all check against it, and CompiledObjects chooses its compiler
* from the same normalised value, so nothing that passes the check can reach the wrong compiler.
*/
- val supportedLanguages: List[String] = List("scala", "java")
+ val supportedLanguages: List[String] = List("scala", "java", "query")
def isSupportedLanguage(programmingLang: String): Boolean =
supportedLanguages.contains(APIUtil.normaliseDynamicCodeLanguage(programmingLang))
- val supportedLanguagesText: String = "Scala, Java"
+ val supportedLanguagesText: String = "Scala, Java, Query"
+
+ /**
+ * True when the body is a Dynamic Query (`programming_lang` `Query`): a declaration, not code. It
+ * runs no user code, so the switch for user-supplied code (`allow_user_generated_scala_code`) does
+ * not apply to it, and it is read-only, so its doc's request_verb must be GET.
+ */
+ def isQuery(programmingLang: String): Boolean = APIUtil.normaliseDynamicCodeLanguage(programmingLang) == "query"
+
+ /** A Dynamic Query only reads; every other language may use any verb. */
+ def verbAllowed(programmingLang: String, requestVerb: String): Boolean =
+ !isQuery(programmingLang) || requestVerb == "GET"
+
+ val queryVerbMessage: String = "A Dynamic Query only reads, so its request_verb must be GET."
/**
* The native http4s template a method body is inlined into. Returns the full source and the
@@ -167,14 +181,26 @@ object CompiledObjects {
* diagnostics with line numbers relative to the method body the author wrote, in the body's
* programming language (Scala or Java). Empty = compiles.
*/
- def compileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String, programmingLang: String = "Scala"): List[DynamicUtil.CompileProblem] = {
+ def compileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String,
+ programmingLang: String = "Scala", bankId: Option[String] = None): List[DynamicUtil.CompileProblem] = {
val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8")
// Java bodies are compiled as written (no template, no generated case classes), so the example
- // bodies play no part, as they play none in CompiledObjects' own Java branch.
- if (APIUtil.normaliseDynamicCodeLanguage(programmingLang) == "java") DynamicUtil.checkJavaCode(decodedMethodBody)
- else scalaCompileProblems(exampleRequestBody, successResponseBody, decodedMethodBody)
+ // bodies play no part, as they play none in CompiledObjects' own Java branch. A Dynamic Query is
+ // not compiled at all: its problems are the declaration's, which have no line numbers.
+ APIUtil.normaliseDynamicCodeLanguage(programmingLang) match {
+ case "java" => DynamicUtil.checkJavaCode(decodedMethodBody)
+ case "query" => queryProblem(decodedMethodBody, bankId).map(message => DynamicUtil.CompileProblem(0, 0, "ERROR", message)).toList
+ case _ => scalaCompileProblems(exampleRequestBody, successResponseBody, decodedMethodBody)
+ }
}
+ /** What is wrong with a Dynamic Query body, in full (error code included), or None when it is valid in `bankId`'s space. */
+ def queryProblem(decodedMethodBody: String, bankId: Option[String]): Option[String] =
+ code.api.dynamic.entity.query.DynamicQueryDeclaration.parse(decodedMethodBody) match {
+ case Left(error) => Some(s"${ErrorMessages.DynamicQueryInvalid}${error.message}")
+ case Right(declaration) => code.api.dynamic.entity.query.DynamicQuery.validate(bankId, declaration).left.toOption.map(_.message)
+ }
+
private def scalaCompileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], decodedMethodBody: String): List[DynamicUtil.CompileProblem] = {
val requestBody: Product = exampleRequestBody match {
case Some(JString(s)) if StringUtils.isBlank(s) => toCaseObject(None)
@@ -205,17 +231,40 @@ object CompiledObjects {
}
}
-case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String, programmingLang: String = "Scala") {
+/**
+ * `bankId` is the Dynamic Entity space of the doc (None for the system space). Only a Dynamic Query
+ * uses it: its declaration names entities, which are looked up in that space.
+ */
+case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String,
+ programmingLang: String = "Scala", bankId: Option[String] = None) {
val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8")
+ private val isQuery = CompiledObjects.isQuery(programmingLang)
+ // The stored doc of a system-level Dynamic Resource Doc carries Some(null) here, not None.
+ private val space: Option[String] = bankId.flatMap(Option(_)).map(_.trim).filter(_.nonEmpty)
+
+ // A Dynamic Query compiles nothing, not even case classes for its examples: toCaseObject generates
+ // and compiles Scala, which is refused where user-supplied code is switched off. Its examples are
+ // carried as JSON, which the resource-docs serialisation renders as they are.
+ private def exampleOf(json: Option[JValue]): Product =
+ if (!isQuery) toCaseObject(json)
+ else json.filter(j => j != JNothing && j != JNull).map(code.api.berlin.group.v1_3.JvalueCaseClass(_)).getOrElse(EmptyBody)
+
val requestBody: Product = exampleRequestBody match {
//this case means, we accept the empty string "" from json post body, we need to map it to None.
- case Some(JString(s)) if StringUtils.isBlank(s) => toCaseObject(None)
+ case Some(JString(s)) if StringUtils.isBlank(s) => exampleOf(None)
// Here we will generate the object by the JValue (exampleRequestBody)
- case _ => toCaseObject(exampleRequestBody)
+ case _ => exampleOf(exampleRequestBody)
}
- val successResponse: Product = toCaseObject(successResponseBody)
+ val successResponse: Product = exampleOf(successResponseBody)
private val partialFunction: Http4sEndpointIO = APIUtil.normaliseDynamicCodeLanguage(programmingLang) match {
+ case "query" =>
+ // A declaration, not code: parsed here (a malformed body cannot be served), checked against the
+ // entity definitions by validateDependency, and run per request by DynamicQueryEndpoint.
+ code.api.dynamic.entity.query.DynamicQueryDeclaration.parse(decodedMethodBody) match {
+ case Right(declaration) => DynamicQueryEndpoint(declaration, space)
+ case Left(error) => throw JsonResponseException(s"${ErrorMessages.DynamicQueryInvalid}${error.message}", 400, "none")
+ }
case "java" =>
DynamicUtil.createJavaHttp4sEndpoint(decodedMethodBody) match {
case Full(func) => func
@@ -286,6 +335,8 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo
*/
def validateDependency() = APIUtil.normaliseDynamicCodeLanguage(programmingLang) match {
case "java" => ()
+ // A Dynamic Query calls no methods; what it is checked against is the entity definitions of its space.
+ case "query" => CompiledObjects.queryProblem(decodedMethodBody, space).foreach(message => throw JsonResponseException(message, 400, "none"))
case _ => Validation.validateDependency(this.partialFunction)
}
@@ -294,7 +345,7 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo
* author wrote (the wrapper's own lines are subtracted). Empty = compiles. Nothing is evaluated or cached.
*/
def compileProblems(): List[DynamicUtil.CompileProblem] =
- CompiledObjects.compileProblems(exampleRequestBody, successResponseBody, methodBody, programmingLang)
+ CompiledObjects.compileProblems(exampleRequestBody, successResponseBody, methodBody, programmingLang, space)
/**
* Wraps the compiled partial function as an endpoint. This used to bind a per-bank
diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicQueryEndpoint.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicQueryEndpoint.scala
new file mode 100644
index 0000000000..d7a986ccf3
--- /dev/null
+++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicQueryEndpoint.scala
@@ -0,0 +1,71 @@
+/**
+Open Bank Project - API
+Copyright (C) 2011-2026, TESOBE GmbH.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+ */
+
+package code.api.dynamic.endpoint.helper
+
+import cats.effect.IO
+import code.api.dynamic.entity.query.{DynamicQuery, DynamicQueryDeclaration}
+import code.api.util.APIUtil.Http4sEndpointIO
+import code.api.util.{APIUtil, CallContext}
+import code.util.Helper.MdcLoggable
+import com.openbankproject.commons.util.JsonAliases.compactRender
+import org.http4s.headers.`Content-Type`
+import org.http4s.{MediaType, Request, Response, Status}
+import org.json4s.JsonAST.{JInt, JObject, JString}
+
+/**
+ * This object turns a Dynamic Query declaration into the handler of a Dynamic Resource Doc whose
+ * `programming_lang` is `Query`. Nothing is compiled: each request is answered by
+ * [[DynamicQuery.run]] for the caller, in the Dynamic Entity space of the doc (`bankId`, None for the
+ * system space). The doc's own Roles are enforced by the middleware before this runs, as for any
+ * Dynamic Resource Doc.
+ */
+object DynamicQueryEndpoint extends MdcLoggable {
+
+ private val jsonContentType = `Content-Type`(MediaType.application.json)
+
+ def apply(declaration: DynamicQueryDeclaration, bankId: Option[String]): Http4sEndpointIO = new Http4sEndpointIO {
+ override def isDefinedAt(req: Request[IO]): Boolean = true
+
+ override def apply(req: Request[IO]): CallContext => IO[Response[IO]] = { cc =>
+ val callerParams = req.uri.query.multiParams.map { case (name, values) => name -> values.toList }
+ IO.blocking {
+ DynamicQuery.run(bankId, declaration, callerParams, cc.user.map(_.userId).toOption, APIUtil.getConsumerPrimaryKey(Some(cc)))
+ }.map {
+ case Right(result) => json(Status.Ok, result)
+ case Left(failure) => json(Status.fromInt(failure.status).getOrElse(Status.BadRequest),
+ JObject(List("code" -> JInt(failure.status), "message" -> JString(failure.message))))
+ }.handleError { e =>
+ logger.warn(s"DynamicQueryEndpoint says: the Dynamic Query from '${declaration.from}' failed", e)
+ json(Status.InternalServerError, JObject(List("code" -> JInt(500), "message" -> JString(s"OBP-50000: Unknown Error. ${e.getMessage}"))))
+ }
+ }
+ }
+
+ private def json(status: Status, body: JObject): Response[IO] =
+ Response[IO](status).withEntity(compactRender(body)).withContentType(jsonContentType)
+}
diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala
index 2c52f6eb8e..7ea394c1a2 100644
--- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala
+++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala
@@ -64,9 +64,11 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel
case APIUtil.JsonResponseExtractor(msg, _) => msg
case _ => Option(e.getMessage).getOrElse("")
}
+ val rejectedBy =
+ if (CompiledObjects.isQuery(dynamicDoc.programmingLang)) "its Dynamic Query declaration is not valid"
+ else "rejected by dependency validation (dynamic_code_allowed_obp_methods)"
logger.error(s"[DynamicResourceDocsEndpointGroup] skipping dynamic resource doc '${dynamicDoc.requestVerb} ${dynamicDoc.requestUrl}' " +
- s"(id=${dynamicDoc.dynamicResourceDocId.getOrElse("")}, programming_lang=${dynamicDoc.programmingLang}): rejected by dependency " +
- s"validation (dynamic_code_allowed_obp_methods). $reason")
+ s"(id=${dynamicDoc.dynamicResourceDocId.getOrElse("")}, programming_lang=${dynamicDoc.programmingLang}): $rejectedBy. $reason")
None
case e: Throwable =>
logger.error(s"[DynamicResourceDocsEndpointGroup] skipping dynamic resource doc '${dynamicDoc.requestVerb} ${dynamicDoc.requestUrl}' " +
@@ -96,7 +98,8 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel
*
*/
private val toResourceDoc: JsonDynamicResourceDoc => ResourceDoc = { dynamicDoc =>
- val compiledObjects = CompiledObjects(dynamicDoc.exampleRequestBody, dynamicDoc.successResponseBody, dynamicDoc.methodBody, dynamicDoc.programmingLang)
+ val compiledObjects = CompiledObjects(dynamicDoc.exampleRequestBody, dynamicDoc.successResponseBody, dynamicDoc.methodBody,
+ dynamicDoc.programmingLang, dynamicDoc.bankId)
ResourceDoc(
// partialFunction is a no-op stub — the runtime dispatch uses the native handler in
// dynamicHttp4sFunction (the compiled artifact is OBPEndpointIO, not the Lift OBPEndpoint).
@@ -117,7 +120,9 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel
StringUtils.split(it, ",")
.map(ApiRole.getOrCreateDynamicApiRole(_))
.toList
- }
+ },
+ // The bank level resource-docs endpoints list a space's docs by this field.
+ createdByBankId = dynamicDoc.bankId.flatMap(Option(_)).filter(_.nonEmpty)
)
}
}
diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala b/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala
index deaa70aabe..897375e3f2 100644
--- a/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala
+++ b/obp-api/src/main/scala/code/api/dynamic/entity/Http4sDynamicEntity.scala
@@ -31,7 +31,7 @@ import code.DynamicData.{DynamicData, DynamicDataProvider, DynamicDataAccessProv
import code.api.Constant.PARAM_LOCALE
import code.api.dynamic.entity.helper.{CommunityEntityName, DynamicEntityHelper, DynamicEntityInfo, DynamicEntitySpace, EntityAccessName, EntityName, PublicEntityName}
import code.api.dynamic.entity.query.{FieldSpec, InMemoryQueryExecutor, JoinTargetInfo, QueryParamParser, QueryPlan, QueryPlanner}
-import code.api.dynamic.entity.projection.{IndexingCapabilities, PostgresProjectionBackend, ProjectionProvisioner}
+import code.api.dynamic.entity.projection.{IndexingCapabilities, PostgresProjectionBackend, ProjectionReadiness}
import cats.effect.unsafe.implicits.{global => ioRuntime} // aliased: avoids clashing with the EC `global` imported below
import code.api.util.APIUtil._
import code.api.util.ErrorMessages._
@@ -414,19 +414,34 @@ object Http4sDynamicEntity extends MdcLoggable {
case other => other
}
- // Remove any read-restricted field the caller lacks the read role for (anonymous => userIdOpt None => omit all).
- private def applyReadRestrictions(value: JValue, bankId: Option[String], entityName: String, userIdOpt: Option[String]): JValue = {
+ // Remove any read-restricted field the caller lacks the read role for (anonymous => userIdOpt None => omit all),
+ // and, on a read through public access, every field declared hide_field_from_public_access.
+ private def applyReadRestrictions(value: JValue, bankId: Option[String], entityName: String, userIdOpt: Option[String],
+ viaPublicAccess: Boolean = false): JValue = {
val info = DynamicEntityHelper.definitionOf(bankId, entityName)
- val readRestricted = info.map(_.readRestrictedFields).getOrElse(Nil)
- val omit: Set[String] = readRestricted.filterNot { f =>
- userIdOpt.exists { uid =>
- val role = DynamicEntityInfo.fieldReadRole(entityName, f, bankId, info.flatMap(_.explicitReadRole(f)))
- code.api.util.APIUtil.hasEntitlement(DynamicEntitySpace.bankIdOrSystem(bankId), uid, role)
- }
- }.toSet
+ val restricted = info.map(_.readRestrictedFields).getOrElse(Nil) ++ info.map(_.publicHiddenFields).getOrElse(Nil)
+ val omit: Set[String] = restricted.filterNot(f => DynamicEntityInfo.mayReadField(bankId, entityName, f, userIdOpt, viaPublicAccess)).toSet
if (omit.isEmpty) value else omitFields(value, omit)
}
+ /**
+ * A caller may not filter or sort on a field they may not read: whether a record comes back would
+ * reveal the field's value. This covers the legacy bare-parameter filter (`?field=value`),
+ * `obp_filter`, `obp_sort_by`, and the nested filters of `obp_exists` / `obp_not_exists`, which are
+ * judged on the joined entity. `viaPublicAccess` says whether this read of `entityName` is through
+ * its public access, which hides the fields declared `hide_field_from_public_access`.
+ */
+ private def refuseUnreadableQueryFields(req: Request[IO], plan: QueryPlan, bankId: Option[String], entityName: String,
+ userIdOpt: Option[String], viaPublicAccess: Boolean, cc: Option[CallContext]): Future[Box[Unit]] = {
+ val joinedEntityReader = DynamicEntityInfo.fieldReader(bankId, userIdOpt, code.api.util.APIUtil.getConsumerPrimaryKey(cc))
+ val legacyFields = queryParams(req).keys.filter(k => k != PARAM_LOCALE && !k.startsWith("obp_")).map(_.takeWhile(_ != '.'))
+ val ownFields = (legacyFields ++ ProjectionReadiness.planFields(plan)).toList.distinct
+ .filterNot(f => DynamicEntityInfo.mayReadField(bankId, entityName, f, userIdOpt, viaPublicAccess)).map(f => s"$entityName.$f")
+ val joinedFields = plan.joins.flatMap(j => j.predicate.map(_.field).distinct.filterNot(joinedEntityReader(j.childEntity, _)).map(f => s"${j.childEntity}.$f"))
+ val unreadable = ownFields ++ joinedFields
+ Helper.booleanToFuture(s"$DynamicEntityFieldNotReadable${unreadable.mkString(", ")}.", 400, cc = cc) { unreadable.isEmpty }
+ }
+
// ----- DE_indexing: read-path backend selection (projection vs in-memory) -----
// Phase 3: applied to the authenticated genericGet only. Public/community stay in-memory for now
// (different scoping). Projection is used only for pure obp_filter/sort queries (no legacy bare
@@ -445,28 +460,15 @@ object Http4sDynamicEntity extends MdcLoggable {
private def joinParamsPresent(req: Request[IO]): Boolean =
queryParams(req).keys.exists(k => k.startsWith("obp_exists[") || k.startsWith("obp_not_exists["))
- private def planFields(plan: QueryPlan): List[String] =
- (plan.filters.map(_.field) ++ plan.sort.map(_.field)).distinct
-
private def decideProjection(req: Request[IO], bankId: Option[String], entityName: String, plan: QueryPlan): ProjDecision =
if (plan.joins.nonEmpty) {
// Joins are projection-only. Legacy bare params can't combine with joins (they force in-memory).
if (!IndexingCapabilities.projectionEnabled || legacyParamsPresent(req)) JoinsNeedProjection
- else {
- val parentReady = ProjectionProvisioner.readyFields(bankId, entityName)
- val parentFieldsReady = planFields(plan).forall(parentReady.contains)
- val joinsReady = plan.joins.forall { j =>
- val childReady = ProjectionProvisioner.readyFields(bankId, j.childEntity)
- val linkReady = if (j.onChild) childReady.contains(j.linkField) else parentReady.contains(j.linkField)
- linkReady && j.predicate.map(_.field).forall(childReady.contains)
- }
- if (parentFieldsReady && joinsReady) UseProjection else PendingProjection
- }
- } else if (!IndexingCapabilities.projectionEnabled || legacyParamsPresent(req) || planFields(plan).isEmpty) UseInMemory
- else {
- val ready = ProjectionProvisioner.readyFields(bankId, entityName)
- if (planFields(plan).forall(ready.contains)) UseProjection else PendingProjection
- }
+ else if (ProjectionReadiness.ready(bankId, entityName, plan)) UseProjection
+ else PendingProjection
+ } else if (!IndexingCapabilities.projectionEnabled || legacyParamsPresent(req) || ProjectionReadiness.planFields(plan).isEmpty) UseInMemory
+ else if (ProjectionReadiness.ready(bankId, entityName, plan)) UseProjection
+ else PendingProjection
private def projectionList(entityName: String, bankId: Option[String], userId: Option[String], isPersonalEntity: Boolean, plan: QueryPlan): Future[JArray] =
PostgresProjectionBackend.query(entityName, bankId, userId, isPersonalEntity, plan).map(JArray(_)).unsafeToFuture()(ioRuntime)
@@ -497,6 +499,8 @@ object Http4sDynamicEntity extends MdcLoggable {
// Row-level get-all is served in-memory (ACL-gated); joins require the projection backend.
_ <- if (isGetAll && joinParamsPresent(req)) Helper.booleanToFuture(DynamicEntityJoinRequiresProjection, 400, cc = callContext) { false }
else Future.successful(true)
+ _ <- if (isGetAll) refuseUnreadableQueryFields(req, QueryPlan.empty, bankId, entityName, Some(u.userId), viaPublicAccess = false, callContext)
+ else Future.successful(true)
result <- if (isGetAll) Future {
// In-memory floor: fetch all rows (unscoped) and keep those the ACL marks readable.
// (The projection EXISTS backend for row-level get-all is a documented follow-up; the
@@ -681,6 +685,8 @@ object Http4sDynamicEntity extends MdcLoggable {
else checkEntityRole(bankId, entityName, boxUser, DynamicEntityInfo.canGetRole(entityName, bankId), callContext)
_ <- failIf(afterIntercept(callContext, operationId), callContext)
queryPlan <- if (isGetAll) buildQueryPlan(req, bankId, entityName, callContext) else Future.successful(QueryPlan.empty)
+ _ <- if (isGetAll) refuseUnreadableQueryFields(req, queryPlan, bankId, entityName, userIdOpt, viaPublicAccess = false, callContext)
+ else Future.successful(true)
decision = if (isGetAll) decideProjection(req, bankId, entityName, queryPlan) else UseInMemory
_ <- if (decision == JoinsNeedProjection) Helper.booleanToFuture(DynamicEntityJoinRequiresProjection, 400, cc = callContext) { false }
else Future.successful(true)
@@ -840,6 +846,8 @@ object Http4sDynamicEntity extends MdcLoggable {
(_, callContext) <- anonymousAccess(callContext0)
(_, callContext) <- bankCheck(bankId, callContext)
queryPlan <- if (isGetAll) buildQueryPlan(req, bankId, entityName, callContext) else Future.successful(QueryPlan.empty)
+ _ <- if (isGetAll) refuseUnreadableQueryFields(req, queryPlan, bankId, entityName, None, viaPublicAccess = true, callContext)
+ else Future.successful(true)
// Public reads are in-memory only; joins require the projection backend.
_ <- if (queryPlan.joins.nonEmpty) Helper.booleanToFuture(DynamicEntityJoinRequiresProjection, 400, cc = callContext) { false }
else Future.successful(true)
@@ -850,10 +858,10 @@ object Http4sDynamicEntity extends MdcLoggable {
val resultList: JArray = unboxResult(box.asInstanceOf[Box[JArray]], entityName)
val legacyFiltered = filterDynamicObjects(resultList, queryParams(req))
val filtered = applyQueryPlan(legacyFiltered, queryPlan, deIndexedFields(bankId, entityName))
- listResponse(req, bankId, entityName, applyReadRestrictions(filtered, bankId, entityName, None), showUserIds = false)
+ listResponse(req, bankId, entityName, applyReadRestrictions(filtered, bankId, entityName, None, viaPublicAccess = true), showUserIds = false)
} else {
val singleObject: JValue = unboxResult(box.asInstanceOf[Box[JValue]], entityName)
- singleResponse(req, bankId, entityName, applyReadRestrictions(singleObject, bankId, entityName, None), showUserIds = false)
+ singleResponse(req, bankId, entityName, applyReadRestrictions(singleObject, bankId, entityName, None, viaPublicAccess = true), showUserIds = false)
}
}
}
@@ -873,6 +881,8 @@ object Http4sDynamicEntity extends MdcLoggable {
_ <- NewStyle.function.hasEntitlement(DynamicEntitySpace.bankIdOrSystem(bankId), u.userId, DynamicEntityInfo.canGetRole(entityName, bankId), callContext)
_ <- failIf(afterIntercept(callContext, operationId), callContext)
queryPlan <- if (isGetAll) buildQueryPlan(req, bankId, entityName, callContext) else Future.successful(QueryPlan.empty)
+ _ <- if (isGetAll) refuseUnreadableQueryFields(req, queryPlan, bankId, entityName, Some(u.userId), viaPublicAccess = false, callContext)
+ else Future.successful(true)
// Community reads are in-memory only; joins require the projection backend.
_ <- if (queryPlan.joins.nonEmpty) Helper.booleanToFuture(DynamicEntityJoinRequiresProjection, 400, cc = callContext) { false }
else Future.successful(true)
diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala
index 71c5b41a5e..1de39d0c0c 100644
--- a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala
+++ b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala
@@ -150,9 +150,11 @@ object DynamicEntityHelper {
/**
* DE_indexing: may this definition update be applied to an entity that already has rows?
*
- * The stored rows stay valid when the entity name, the set of property names and each property's `type`
- * are unchanged and `required` does not grow. Everything else — `indexed`, `index`, `example`,
- * `description`, `minLength`, `maxLength`, `read_role*`, `write_role*` — may change freely; in
+ * The stored rows stay valid when the entity name is unchanged, every existing property keeps its name and
+ * `type`, and `required` does not grow. A new property may be added: the stored rows simply don't have it,
+ * which is valid as long as it is optional (a new required property grows `required`, so is refused).
+ * Removing a property or changing its type is refused. Everything else — `indexed`, `index`, `example`,
+ * `description`, `minLength`, `maxLength`, `read_role*`, `write_role*`, `hide_field_from_public_access` — may change freely; in
* particular this is what lets an operator switch indexing on for an existing, populated entity
* (the projection backfill then does the rest). Unparseable input is treated as incompatible.
*/
@@ -186,7 +188,9 @@ object DynamicEntityHelper {
oldEntityName == newEntityName && {
(definitionOf(oldMetadataJson, oldEntityName), definitionOf(newMetadataJson, newEntityName)) match {
case (Some(oldDef), Some(newDef)) =>
- propertyTypes(oldDef) == propertyTypes(newDef) && requiredNames(newDef).subsetOf(requiredNames(oldDef))
+ val newTypes = propertyTypes(newDef)
+ propertyTypes(oldDef).forall { case (name, oldType) => newTypes.get(name).contains(oldType) } &&
+ requiredNames(newDef).subsetOf(requiredNames(oldDef))
case _ => false
}
}
@@ -1121,6 +1125,17 @@ case class DynamicEntityInfo(definition: String, entityName: String, bankId: Opt
lazy val writeRestrictedFields: List[String] = restrictedFields("write_role_required", "write_role")
/** Fields omitted from GET unless the caller holds the read role. */
lazy val readRestrictedFields: List[String] = restrictedFields("read_role_required", "read_role")
+ /**
+ * Fields declared `"hide_field_from_public_access": true`: hidden from a caller whose access to this
+ * entity comes only from its public access, shown to a caller with other access (its read Role, or a
+ * row-level access list entry). See DynamicEntityInfo.readsViaPublicAccess.
+ */
+ lazy val publicHiddenFields: List[String] = (entity \ "properties") match {
+ case props: JObject => props.obj.collect {
+ case JField(name, propDef: JObject) if (propDef \ "hide_field_from_public_access") == JBool(true) => name
+ }
+ case _ => Nil
+ }
def explicitWriteRole(fieldName: String): Option[String] =
(entity \ "properties" \ fieldName \ "write_role") match { case JString(s) if s.nonEmpty => Some(s); case _ => None }
def explicitReadRole(fieldName: String): Option[String] =
@@ -1150,6 +1165,24 @@ case class DynamicEntityInfo(definition: String, entityName: String, bankId: Opt
case _ => Map.empty
}
+ /**
+ * Every declared property whose type is a scalar type the query layer understands: name -> type.
+ * A `reference:` field reads as [[DynamicEntityFieldType.reference]]. Unlike [[indexedFields]]
+ * this covers fields whether or not they are indexed; a join's `where` filter and `pick` order are
+ * evaluated in memory on the fetched records, so they may use any of these fields.
+ */
+ lazy val declaredFieldTypes: Map[String, DynamicEntityFieldType] = (entity \ "properties") match {
+ case props: JObject => props.obj.collect {
+ case JField(name, propDef: JObject) =>
+ val typeName = (propDef \ "type") match { case JString(s) => s; case _ => "" }
+ val fieldTypeOpt =
+ if (typeName.startsWith("reference:")) Some(DynamicEntityFieldType.reference)
+ else DynamicEntityFieldType.withNameOption(typeName)
+ fieldTypeOpt.map(name -> _)
+ }.flatten.toMap
+ case _ => Map.empty
+ }
+
/**
* Every `reference:` field, indexed or not: fieldName -> target entity name (the part after
* "reference:"). Only the indexed subset ([[referenceFields]]) forms a join edge; the rest
@@ -1291,4 +1324,60 @@ object DynamicEntityInfo {
case Some(role) => getOrCreateDynamicApiRole(role, true)
case None => getOrCreateDynamicApiRole(s"CanGetDynamicEntityField_${entityName}__${fieldName}", true)
}
+
+ /**
+ * This says whether a caller may see one field of an entity's records. A field declared
+ * `read_role_required` is shown only to a caller holding its read Role in the entity's space; an
+ * anonymous caller (None) never sees it. Every other field is readable. It is the per-field rule
+ * that a GET applies when it omits restricted fields, and that a join applies when it copies
+ * a field from a referenced record.
+ */
+ /**
+ * This says whether a caller may read an entity's shared records, by the same rule a GET applies:
+ * an entity with public access is readable by anyone; a row-level entity is readable in principle,
+ * because its access list then decides row by row; otherwise the caller needs the entity's get
+ * Role in its space, judged by the entity's authentication mode (so a Consumer can qualify where
+ * the mode allows it). An unknown entity is not readable.
+ */
+ /**
+ * True when a caller's access to an entity comes only from its public access: the entity has public
+ * access, the caller does not hold its read Role (judged by its authentication mode), and the caller is
+ * not reaching it through a row-level access list (an anonymous caller never is). A field declared
+ * `hide_field_from_public_access` is hidden from such a caller.
+ */
+ def readsViaPublicAccess(bankId: Option[String], entityName: String, userIdOpt: Option[String], consumerId: String): Boolean =
+ DynamicEntityHelper.definitionOf(bankId, entityName).exists { info =>
+ info.hasPublicAccess && !(info.useRowLevelAccess && userIdOpt.isDefined) &&
+ !APIUtil.handleAccessControlWithAuthMode(DynamicEntitySpace.bankIdOrSystem(bankId), userIdOpt.getOrElse(""), consumerId,
+ List(canGetRole(entityName, bankId)), info.endpointAuthMode)
+ }
+
+ /**
+ * Whether one caller may read each field, as `(entity, field) => Boolean`, for reads that span
+ * several entities (joins, Dynamic Queries): [[mayReadField]], with each entity's public-access
+ * question answered once per entity.
+ */
+ def fieldReader(bankId: Option[String], userIdOpt: Option[String], consumerId: String): (String, String) => Boolean = {
+ val viaPublic = scala.collection.mutable.Map[String, Boolean]()
+ (entityName, fieldName) => mayReadField(bankId, entityName, fieldName, userIdOpt,
+ viaPublic.getOrElseUpdate(entityName, readsViaPublicAccess(bankId, entityName, userIdOpt, consumerId)))
+ }
+
+ def mayReadRecords(bankId: Option[String], entityName: String, userIdOpt: Option[String], consumerId: String): Boolean =
+ DynamicEntityHelper.definitionOf(bankId, entityName).exists { info =>
+ info.hasPublicAccess || info.useRowLevelAccess ||
+ APIUtil.handleAccessControlWithAuthMode(DynamicEntitySpace.bankIdOrSystem(bankId), userIdOpt.getOrElse(""), consumerId,
+ List(canGetRole(entityName, bankId)), info.endpointAuthMode)
+ }
+
+ def mayReadField(bankId: Option[String], entityName: String, fieldName: String, userIdOpt: Option[String],
+ viaPublicAccess: Boolean = false): Boolean = {
+ val info = DynamicEntityHelper.definitionOf(bankId, entityName)
+ if (viaPublicAccess && info.exists(_.publicHiddenFields.contains(fieldName))) false
+ else if (!info.exists(_.readRestrictedFields.contains(fieldName))) true
+ else userIdOpt.exists { userId =>
+ val role = fieldReadRole(entityName, fieldName, bankId, info.flatMap(_.explicitReadRole(fieldName)))
+ APIUtil.hasEntitlement(DynamicEntitySpace.bankIdOrSystem(bankId), userId, role)
+ }
+ }
}
diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/projection/PostgresProjectionBackend.scala b/obp-api/src/main/scala/code/api/dynamic/entity/projection/PostgresProjectionBackend.scala
index d0cc7f4bac..7be2a02979 100644
--- a/obp-api/src/main/scala/code/api/dynamic/entity/projection/PostgresProjectionBackend.scala
+++ b/obp-api/src/main/scala/code/api/dynamic/entity/projection/PostgresProjectionBackend.scala
@@ -52,7 +52,34 @@ object PostgresProjectionBackend extends DynamicEntityQueryBackend {
def name: String = "postgres-projection"
- def query(entityName: String, bankId: Option[String], userId: Option[String], isPersonalEntity: Boolean, plan: QueryPlan): IO[List[JObject]] = {
+ def query(entityName: String, bankId: Option[String], userId: Option[String], isPersonalEntity: Boolean, plan: QueryPlan): IO[List[JObject]] =
+ statement(entityName, bankId, userId, isPersonalEntity, plan, counting = false) match {
+ case Some(q) => ProjectionDb.run(q.query[String].to[List]).map(_.map(s => com.openbankproject.commons.util.JsonAliases.parse(s).asInstanceOf[JObject]))
+ case None => IO.raiseError(new RuntimeException(s"PostgresProjectionBackend: unresolved field in query plan for $entityName"))
+ }
+
+ /**
+ * How many records the plan's filters and joins match, ignoring its sort and page: the total a
+ * paged response can report alongside one page. Same statement as [[query]], counted.
+ */
+ def count(entityName: String, bankId: Option[String], userId: Option[String], isPersonalEntity: Boolean, plan: QueryPlan): IO[Long] =
+ statement(entityName, bankId, userId, isPersonalEntity, plan, counting = true) match {
+ case Some(q) => ProjectionDb.run(q.query[Long].unique)
+ case None => IO.raiseError(new RuntimeException(s"PostgresProjectionBackend: unresolved field in query plan for $entityName"))
+ }
+
+ /**
+ * The SQL text [[query]] (or, with `counting`, [[count]]) would send for this plan, with `?` for every
+ * bound value, without running it. None if a field cannot resolve. This is what an explain facility
+ * shows: the same builder as the statement that runs, so the two cannot differ.
+ */
+ def sqlFor(entityName: String, bankId: Option[String], userId: Option[String], isPersonalEntity: Boolean,
+ plan: QueryPlan, counting: Boolean = false): Option[String] =
+ statement(entityName, bankId, userId, isPersonalEntity, plan, counting).map(_.query[String].sql)
+
+ /** The SELECT for a plan: the matching blobs in order and paged, or (`counting`) their number. None if a field cannot resolve. */
+ private def statement(entityName: String, bankId: Option[String], userId: Option[String], isPersonalEntity: Boolean,
+ plan: QueryPlan, counting: Boolean): Option[Fragment] = {
val indexed = DynamicEntityHelper.definitionOf(bankId, entityName).map(_.indexedFields).getOrElse(Map.empty)
val safeTable = ProjectionNaming.tableName(bankId, entityName)
val P = "p"; val D = "d"
@@ -72,15 +99,14 @@ object PostgresProjectionBackend extends DynamicEntityQueryBackend {
val whereAll =
if (condParts.isEmpty) fr"WHERE" ++ scope
else fr"WHERE" ++ scope ++ fr"AND" ++ ProjectionSql.intercalate(condParts, fr"AND")
- val q =
- fr"SELECT" ++ Fragment.const(s"$D.${ProjectionStore.jsonColumn}") ++
+ val selected = if (counting) fr"count(*)" else Fragment.const(s"$D.${ProjectionStore.jsonColumn}")
+ Some(
+ fr"SELECT" ++ selected ++
fr"FROM" ++ Fragment.const(s"$safeTable $P") ++
fr"JOIN" ++ Fragment.const(s"${ProjectionStore.blobTable} $D") ++
fr"ON" ++ Fragment.const(s"$D.${ProjectionStore.idColumn} = $P.data_id") ++
- whereAll ++ ords ++ ProjectionSql.limitOffset(plan)
- ProjectionDb.run(q.query[String].to[List]).map(_.map(s => com.openbankproject.commons.util.JsonAliases.parse(s).asInstanceOf[JObject]))
- case _ =>
- IO.raiseError(new RuntimeException(s"PostgresProjectionBackend: unresolved field in query plan for $entityName"))
+ whereAll ++ (if (counting) Fragment.empty else ords ++ ProjectionSql.limitOffset(plan)))
+ case _ => None
}
}
@@ -123,10 +149,15 @@ object PostgresProjectionBackend extends DynamicEntityQueryBackend {
}
}
- /** ACL restriction for a row-level child: only rows the caller can read count toward EXISTS / NOT EXISTS. */
+ /**
+ * ACL restriction for a row-level child: only rows the caller can read count toward EXISTS / NOT EXISTS.
+ * With no caller, no row of a row-level child is readable, so none counts: this fails closed rather than
+ * counting every row, which would reveal that rows the caller cannot read exist.
+ */
private def childAclFragment(childEntity: String, bankId: Option[String], childBlobAlias: String, callerUserId: Option[String]): Fragment = {
val isRowLevel = DynamicEntityHelper.definitionOf(bankId, childEntity).exists(_.useRowLevelAccess)
(isRowLevel, callerUserId) match {
+ case (true, None) => fr"AND FALSE"
case (true, Some(uid)) =>
fr"AND EXISTS (SELECT 1 FROM" ++ Fragment.const(s"${ProjectionStore.aclTable} acl") ++
fr"WHERE" ++ Fragment.const(s"acl.${ProjectionStore.aclDataIdColumn} = $childBlobAlias.${ProjectionStore.idColumn}") ++
diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionReadiness.scala b/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionReadiness.scala
new file mode 100644
index 0000000000..f9c7cc404d
--- /dev/null
+++ b/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionReadiness.scala
@@ -0,0 +1,53 @@
+/**
+Open Bank Project - API
+Copyright (C) 2011-2026, TESOBE GmbH.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+ */
+
+package code.api.dynamic.entity.projection
+
+import code.api.dynamic.entity.query.QueryPlan
+
+/**
+ * This object says whether the query projection can serve a plan right now: every field the plan
+ * filters or sorts on has a projection column that is provisioned and backfilled ("ready"), and so
+ * has the link field and every nested-filter field of each `obp_exists` / `obp_not_exists` join.
+ * A field that is declared indexed but not yet ready makes this false; the caller decides what that
+ * means (the list endpoint answers 409; a Dynamic Query reads in memory instead).
+ */
+object ProjectionReadiness {
+
+ /** The parent fields a plan filters or sorts on. */
+ def planFields(plan: QueryPlan): List[String] =
+ (plan.filters.map(_.field) ++ plan.sort.map(_.field)).distinct
+
+ def ready(bankId: Option[String], entityName: String, plan: QueryPlan): Boolean = {
+ val parentReady = ProjectionProvisioner.readyFields(bankId, entityName)
+ planFields(plan).forall(parentReady.contains) && plan.joins.forall { join =>
+ val childReady = ProjectionProvisioner.readyFields(bankId, join.childEntity)
+ val linkReady = if (join.onChild) childReady.contains(join.linkField) else parentReady.contains(join.linkField)
+ linkReady && join.predicate.map(_.field).forall(childReady.contains)
+ }
+ }
+}
diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionStore.scala b/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionStore.scala
index 905770397f..df810515de 100644
--- a/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionStore.scala
+++ b/obp-api/src/main/scala/code/api/dynamic/entity/projection/ProjectionStore.scala
@@ -84,6 +84,31 @@ object ProjectionStore {
fr"FROM" ++ Fragment.const(blobTable) ++ fr"WHERE" ++ scope(bankId, entityName, isPersonalEntity, userId))
.query[(String, String)].to[List]
+ /**
+ * Read (data_id, dataJson) for the shared records of an entity whose reference column `linkColumn`
+ * holds one of `referencedIds`, through the entity's projection table, so the lookup uses the
+ * column's index. Used to find the records that refer to a page of parent records (a reverse join).
+ * `referencedIds` must not be empty; callers batch long lists.
+ */
+ def readByReference(safeTable: String, linkColumn: String, bankId: Option[String], entityName: String,
+ referencedIds: List[String]): ConnectionIO[List[(String, String)]] =
+ readByReferenceStatement(safeTable, linkColumn, bankId, entityName, referencedIds).query[(String, String)].to[List]
+
+ /** The SQL text of [[readByReferenceStatement]], with `?` for every bound value. */
+ def readByReferenceSql(safeTable: String, linkColumn: String, bankId: Option[String], entityName: String, referencedIds: List[String]): String =
+ readByReferenceStatement(safeTable, linkColumn, bankId, entityName, referencedIds).query[(String, String)].sql
+
+ /** The statement [[readByReference]] runs, as a value, so it can be shown as well as run. */
+ def readByReferenceStatement(safeTable: String, linkColumn: String, bankId: Option[String], entityName: String,
+ referencedIds: List[String]): Fragment = {
+ val idList = ProjectionSql.intercalate(referencedIds.map(id => fr0"$id"), fr",")
+ fr"SELECT" ++ Fragment.const(s"d.$idColumn") ++ fr"," ++ Fragment.const(s"d.$jsonColumn") ++
+ fr"FROM" ++ Fragment.const(s"$safeTable p") ++
+ fr"JOIN" ++ Fragment.const(s"$blobTable d") ++ fr"ON" ++ Fragment.const(s"d.$idColumn = p.data_id") ++
+ fr"WHERE" ++ scope(bankId, entityName, isPersonalEntity = false, None, "d") ++
+ fr"AND" ++ Fragment.const(s"p.$linkColumn") ++ fr"IN (" ++ idList ++ fr")"
+ }
+
/**
* Scope predicate mirroring `MappedDynamicDataProvider`'s get-all: entity name always; bankId
* always (a system-level record stores a sentinel, never NULL); personal flag; userId only when
diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/DynamicQuery.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/DynamicQuery.scala
new file mode 100644
index 0000000000..50829e7860
--- /dev/null
+++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/DynamicQuery.scala
@@ -0,0 +1,503 @@
+/**
+Open Bank Project - API
+Copyright (C) 2011-2026, TESOBE GmbH.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+ */
+
+package code.api.dynamic.entity.query
+
+import code.DynamicData.{DynamicDataAccessProvider, DynamicDataProvider}
+import code.api.dynamic.entity.helper.{DynamicEntityHelper, DynamicEntityInfo}
+import code.api.dynamic.entity.projection.{IndexingCapabilities, PostgresProjectionBackend, ProjectionNaming, ProjectionProvisioner, ProjectionReadiness, ProjectionStore}
+import code.api.util.ErrorMessages.{DynamicEntityJoinRequiresProjection, DynamicQueryEntityNotReadable, DynamicQueryInvalid}
+import com.openbankproject.commons.util.JsonAliases
+import net.liftweb.util.StringHelpers
+import org.json4s.JsonAST._
+
+import scala.util.Try
+
+/**
+ * A Dynamic Query is a Dynamic Resource Doc whose body is a declaration rather than code
+ * (`programming_lang` `Query`). It reads the records of one Dynamic Entity, adds the records joined
+ * to them, and returns them in a named list. See ideas/DYNAMIC_QUERIES.md.
+ *
+ * The body is JSON:
+ *
+ * {{{
+ * {
+ * "from": "activity",
+ * "select": ["activity_id", "name", "city"],
+ * "where": { "city": "eq:Berlin" },
+ * "join": [
+ * { "entity": "operator", "on": "operator_id", "fields": { "operator_legal_name": "legal_name" } },
+ * { "entity": "certificate", "on": "activity_id", "cardinality": "at_most_one",
+ * "pick": "latest_by:issue_date", "fields": { "certificate_number": "number" } }
+ * ],
+ * "envelope": { "rows": "activities", "count": "count" }
+ * }
+ * }}}
+ *
+ * - `from` (required): the entity whose records are returned.
+ * - `select`: the fields of those records to return, in this order; all of them when absent.
+ * - `where`: filters on them, in the list endpoint's grammar (`"field": "op:value"`, or a list of
+ * such strings for several filters on one field); the fields must be declared `"indexed": true`.
+ * - `join`: see [[JoinRequest]]; each entry's keys are `entity`, `on`, `direction`, `cardinality`,
+ * `where`, `fields` (an object, result name -> field), `as`, `pick`, `order`, `true_value`,
+ * `false_value`.
+ * - `envelope`: `rows` names the list (by default the entity's list name, as its own endpoints use)
+ * and `count`, when given, names a field holding how many records match in all, not only on this page.
+ *
+ * Unknown keys are rejected, so a misspelt key is reported instead of silently ignored.
+ */
+case class DynamicQueryEnvelope(rows: String, count: Option[String])
+
+case class DynamicQueryDeclaration(
+ from: String,
+ select: Option[List[String]],
+ where: List[Filter],
+ joins: List[JoinRequest],
+ envelope: DynamicQueryEnvelope
+)
+
+object DynamicQueryDeclaration {
+
+ private val topKeys = List("from", "select", "where", "join", "envelope")
+ private val joinKeys = List("entity", "on", "direction", "cardinality", "where", "fields", "as", "pick", "order", "true_value", "false_value")
+
+ /** Parse a body. The message of a Left says what is wrong and where, without the error code. */
+ def parse(body: String): Either[QueryError, DynamicQueryDeclaration] =
+ for {
+ json <- Try(JsonAliases.parse(body)).toOption.collect { case o: JObject => o }
+ .toRight(QueryError("The body of a Dynamic Query must be a JSON object."))
+ _ <- unknownKeys(json, topKeys, "The Dynamic Query")
+ from <- requiredString(json, "from", "The Dynamic Query")
+ select <- optionalStringList(json, "select", "The Dynamic Query")
+ where <- filtersOf(json \ "where", "The Dynamic Query's 'where'")
+ joins <- json \ "join" match {
+ case JNothing | JNull => Right(Nil)
+ case JArray(items) => traverse(items.zipWithIndex)(item => joinOf(item._1, item._2))
+ case _ => Left(QueryError("The Dynamic Query's 'join' must be a list."))
+ }
+ envelope <- envelopeOf(json \ "envelope", from)
+ } yield DynamicQueryDeclaration(from, select, where, joins, envelope)
+
+ private def joinOf(value: JValue, index: Int): Either[QueryError, JoinRequest] = {
+ val where = s"Join ${index + 1}"
+ value match {
+ case join: JObject =>
+ for {
+ _ <- unknownKeys(join, joinKeys, where)
+ entity <- requiredString(join, "entity", where)
+ on <- requiredString(join, "on", where)
+ direction <- optionalString(join, "direction", where)
+ cardinality <- optionalString(join, "cardinality", where)
+ filters <- filtersOf(join \ "where", s"$where's 'where'")
+ fields <- join \ "fields" match {
+ case JNothing | JNull => Right(Nil)
+ case JObject(pairs) => traverse(pairs) {
+ case (name, JString(field)) => Right(name -> field)
+ case (name, _) => Left(QueryError(s"$where's 'fields' must map each result name to a field name; '$name' does not."))
+ }
+ case _ => Left(QueryError(s"$where's 'fields' must be an object mapping result names to field names."))
+ }
+ as <- optionalString(join, "as", where)
+ pick <- optionalString(join, "pick", where)
+ order <- optionalString(join, "order", where)
+ } yield JoinRequest(entity, on, direction, cardinality, filters, fields, as, pick, order,
+ Some(join \ "true_value").filter(_ != JNothing), Some(join \ "false_value").filter(_ != JNothing))
+ case _ => Left(QueryError(s"$where must be an object."))
+ }
+ }
+
+ private def filtersOf(value: JValue, what: String): Either[QueryError, List[Filter]] = value match {
+ case JNothing | JNull => Right(Nil)
+ case JObject(pairs) =>
+ traverse(pairs) {
+ case (field, JString(raw)) => QueryParamParser.parseOneFilter(field, raw).map(List(_))
+ case (field, JArray(items)) => traverse(items) {
+ case JString(raw) => QueryParamParser.parseOneFilter(field, raw)
+ case _ => Left(QueryError(s"$what: each filter on '$field' must be a string such as \"eq:value\"."))
+ }
+ case (field, _) => Left(QueryError(s"$what: the filter on '$field' must be a string such as \"eq:value\", or a list of them."))
+ }.map(_.flatten).left.map(e => if (e.message.startsWith(what)) e else QueryError(s"$what: ${e.message}"))
+ case _ => Left(QueryError(s"$what must be an object mapping field names to filters such as \"eq:value\"."))
+ }
+
+ private def envelopeOf(value: JValue, from: String): Either[QueryError, DynamicQueryEnvelope] = {
+ val defaultRows = StringHelpers.snakify(from).replaceFirst("[-_]*$", "_list")
+ value match {
+ case JNothing | JNull => Right(DynamicQueryEnvelope(defaultRows, None))
+ case envelope: JObject =>
+ for {
+ _ <- unknownKeys(envelope, List("rows", "count"), "The 'envelope'")
+ rows <- optionalString(envelope, "rows", "The 'envelope'")
+ count <- optionalString(envelope, "count", "The 'envelope'")
+ _ <- if (count.isDefined && count == rows.orElse(Some(defaultRows))) Left(QueryError("The 'envelope' cannot use one name for both 'rows' and 'count'."))
+ else Right(())
+ } yield DynamicQueryEnvelope(rows.getOrElse(defaultRows), count)
+ case _ => Left(QueryError("The 'envelope' must be an object with 'rows' and optionally 'count'."))
+ }
+ }
+
+ private def unknownKeys(json: JObject, allowed: List[String], what: String): Either[QueryError, Unit] =
+ json.obj.map(_._1).find(k => !allowed.contains(k))
+ .map(k => QueryError(s"$what has an unknown key '$k'. Allowed keys: ${allowed.mkString(", ")}.")).toLeft(())
+
+ private def requiredString(json: JObject, key: String, what: String): Either[QueryError, String] =
+ optionalString(json, key, what).flatMap(_.toRight(QueryError(s"$what needs '$key'.")))
+
+ private def optionalString(json: JObject, key: String, what: String): Either[QueryError, Option[String]] =
+ json \ key match {
+ case JNothing | JNull => Right(None)
+ case JString(s) if s.trim.nonEmpty => Right(Some(s.trim))
+ case _ => Left(QueryError(s"$what: '$key' must be a non-empty string."))
+ }
+
+ private def optionalStringList(json: JObject, key: String, what: String): Either[QueryError, Option[List[String]]] =
+ json \ key match {
+ case JNothing | JNull => Right(None)
+ case JArray(items) if items.nonEmpty && items.forall { case JString(s) => s.trim.nonEmpty; case _ => false } =>
+ Right(Some(items.collect { case JString(s) => s.trim }))
+ case _ => Left(QueryError(s"$what: '$key' must be a non-empty list of field names."))
+ }
+
+ private def traverse[A, B](xs: List[A])(f: A => Either[QueryError, B]): Either[QueryError, List[B]] =
+ xs.foldRight(Right(Nil): Either[QueryError, List[B]]) { (a, acc) => for { b <- f(a); rest <- acc } yield b :: rest }
+}
+
+/** One entity a Dynamic Query reads, the access it needs, and whether the explained caller has it. */
+case class ExplainedEntity(entity: String, readRole: String, bankId: String, publicAccess: Boolean, rowLevelAccess: Boolean, callerMayRead: Boolean)
+
+/**
+ * One restricted field a Dynamic Query touches: the rule restricting it (`read_role_required`, or
+ * `hide_field_from_public_access`), the Role that lifts it, and whether the explained caller may read it.
+ */
+case class ExplainedField(entity: String, field: String, restriction: String, readRole: String, callerMayRead: Boolean)
+
+/** One read a Dynamic Query makes: what, how (`projection`, `record provider`, or `shared` with an earlier step), and its SQL when OBP builds it. */
+case class ExplainedStep(purpose: String, backend: String, sql: Option[String], notes: List[String])
+
+/** How a Dynamic Query would be answered for one caller: see [[DynamicQuery.explain]]. */
+case class DynamicQueryExplanation(space: String, anonymousCaller: Boolean, callerMayRun: Boolean, refusal: Option[String],
+ entities: List[ExplainedEntity], restrictedFields: List[ExplainedField], rules: List[String],
+ steps: List[ExplainedStep])
+
+/** Why a Dynamic Query could not answer: the HTTP status and the full message, error code included. */
+case class DynamicQueryFailure(status: Int, message: String)
+
+/**
+ * This object checks and runs Dynamic Queries.
+ *
+ * [[validate]] runs when a doc is created, updated, approved or dry-run compiled. It knows no caller,
+ * so it checks only what the declaration says against the entity definitions of its space.
+ *
+ * [[run]] answers one request. In order:
+ * 1. the caller must be able to read every entity the query reads (`from`, each join, and each
+ * entity a caller's `obp_exists` names), by the rule a GET applies, or the answer is 403; this is
+ * on top of any Roles the doc itself requires;
+ * 2. the caller may narrow the result with the list endpoint's own parameters (`obp_filter`,
+ * `obp_sort_by`, `obp_sort_direction`, `obp_limit`, `obp_offset`, `obp_exists`,
+ * `obp_not_exists`); they are added to the declaration's `where`, never replace it; a filter or
+ * sort on a field the caller may not read is refused;
+ * 3. the page is read: from the query projection when it is enabled and every field the plan needs
+ * is ready, otherwise in memory (`obp_exists` joins need the projection); always shared records
+ * only, and for an entity with row-level access only the records the caller's access list allows;
+ * 4. the joins are applied to the page (see [[RecordJoiner]]);
+ * 5. the fields are chosen (`select`, or all), with a read-restricted field null (when selected) or
+ * left out (when not) unless the caller holds its read Role;
+ * 6. the result is wrapped in the envelope, with the total count when the envelope names one.
+ */
+object DynamicQuery {
+
+ private def invalid(message: String) = DynamicQueryFailure(400, s"$DynamicQueryInvalid$message")
+
+ def validate(bankId: Option[String], declaration: DynamicQueryDeclaration): Either[DynamicQueryFailure, Unit] = {
+ def infoOf(entityName: String): Option[JoinEntityInfo] = DynamicEntityHelper.definitionOf(bankId, entityName).map(JoinEntityInfo.of)
+ for {
+ parentDefinition <- DynamicEntityHelper.definitionOf(bankId, declaration.from)
+ .toRight(invalid(s"There is no Dynamic Entity '${declaration.from}' in this space."))
+ parent = JoinEntityInfo.of(parentDefinition)
+ _ <- selectError(declaration, parent).toLeft(())
+ _ <- planPage(bankId, declaration, parentDefinition, Map.empty).map(_ => ())
+ _ <- JoinPlanner.plan(declaration.from, parent, declaration.joins, infoOf, _ => true, (_, _) => true).left.map(e => invalid(e.message))
+ } yield ()
+ }
+
+ def run(bankId: Option[String], declaration: DynamicQueryDeclaration, callerParams: Map[String, List[String]],
+ callerUserId: Option[String], consumerId: String): Either[DynamicQueryFailure, JObject] = {
+ val from = declaration.from
+ def mayReadEntity(entityName: String): Boolean = DynamicEntityInfo.mayReadRecords(bankId, entityName, callerUserId, consumerId)
+ val mayReadField: (String, String) => Boolean = DynamicEntityInfo.fieldReader(bankId, callerUserId, consumerId)
+ for {
+ parentDefinition <- DynamicEntityHelper.definitionOf(bankId, from).toRight(invalid(s"There is no Dynamic Entity '$from' in this space."))
+ plan <- planPage(bankId, declaration, parentDefinition, callerParams)
+ _ <- accessProblem(bankId, declaration, plan, mayReadEntity, mayReadField).toLeft(())
+ joinPlan <- JoinPlanner.planFor(bankId, from, declaration.joins, callerUserId, consumerId, mayReadEntity).left.map(e => invalid(e.message))
+ pageAndTotal <- readPage(bankId, from, parentDefinition, plan, callerUserId, declaration.envelope.count.isDefined)
+ } yield {
+ val (page, total) = pageAndTotal
+ val joined = joinPlan(page, bankId, callerUserId, consumerId)
+ val joinedNames = joinPlan.joins.flatMap(_.resultNames)
+ val rows = joined.map(record => project(record, declaration.select, joinedNames, parentDefinition, mayReadField(from, _)))
+ JObject(JField(declaration.envelope.rows, JArray(rows)) :: declaration.envelope.count.map(name => JField(name, JInt(total))).toList)
+ }
+ }
+
+ /**
+ * This explains how a Dynamic Query would be answered for one caller, without reading any record:
+ * the statements it would run, in order (the SQL, with `?` for every value, when the step is SQL
+ * built by OBP; words when it goes through the record provider), and the access it needs, with
+ * whether this caller has it. It is for the author of a query, to check that the SQL is sane and
+ * that the access rules are what they expect.
+ *
+ * The SQL comes from the same builders as the statements a call runs, and the verdict from the same
+ * access check ([[accessProblem]]), so the explanation cannot differ from what a call does.
+ */
+ def explain(bankId: Option[String], declaration: DynamicQueryDeclaration, callerParams: Map[String, List[String]],
+ callerUserId: Option[String], consumerId: String): Either[DynamicQueryFailure, DynamicQueryExplanation] = {
+ val from = declaration.from
+ val space = code.api.dynamic.entity.helper.DynamicEntitySpace.bankIdOrSystem(bankId)
+ def definition(entityName: String): Option[DynamicEntityInfo] = DynamicEntityHelper.definitionOf(bankId, entityName)
+ def infoOf(entityName: String): Option[JoinEntityInfo] = definition(entityName).map(JoinEntityInfo.of)
+ def mayReadEntity(entityName: String): Boolean = DynamicEntityInfo.mayReadRecords(bankId, entityName, callerUserId, consumerId)
+ val mayReadField: (String, String) => Boolean = DynamicEntityInfo.fieldReader(bankId, callerUserId, consumerId)
+ for {
+ // The same checks as creating or Check: an invalid declaration is refused, not explained.
+ _ <- validate(bankId, declaration)
+ parentDefinition <- definition(from).toRight(invalid(s"There is no Dynamic Entity '$from' in this space."))
+ plan <- planPage(bankId, declaration, parentDefinition, callerParams)
+ // Planned without the caller's access, so the joins can be described even for a caller who may not run them.
+ joins <- JoinPlanner.plan(from, JoinEntityInfo.of(parentDefinition), declaration.joins, infoOf, _ => true, (_, _) => true)
+ .left.map(e => invalid(e.message))
+ } yield {
+ val entities = entitiesRead(declaration, plan).map { entity =>
+ val info = definition(entity)
+ ExplainedEntity(entity, DynamicEntityInfo.canGetRole(entity, bankId).toString, space,
+ info.exists(_.hasPublicAccess), info.exists(_.useRowLevelAccess), mayReadEntity(entity))
+ }
+ // Read-restricted fields the query touches: those it returns or copies, and those it filters, sorts or picks by.
+ val touched: List[(String, String)] =
+ (declaration.select.getOrElse(parentDefinition.propertyNames).map(from -> _) ++ ProjectionReadiness.planFields(plan).map(from -> _) ++
+ joins.flatMap(j => (j.fields.map(_._2) ++ j.where.map(_.field) ++ j.order.map(_.field).toList).map(j.entity -> _)) ++
+ plan.joins.flatMap(j => j.predicate.map(_.field).map(j.childEntity -> _))).distinct
+ // A field can be restricted twice: by its own read Role, and (for a caller reaching the entity through
+ // public access) by hide_field_from_public_access, which the entity's read Role lifts.
+ val restricted = touched.flatMap { case (entity, field) =>
+ val info = definition(entity)
+ val byRole = info.filter(_.readRestrictedFields.contains(field)).map(i =>
+ ExplainedField(entity, field, "read_role_required",
+ DynamicEntityInfo.fieldReadRole(entity, field, bankId, i.explicitReadRole(field)).toString, mayReadField(entity, field)))
+ val fromPublic = info.filter(_.publicHiddenFields.contains(field)).map(_ =>
+ ExplainedField(entity, field, "hide_field_from_public_access",
+ DynamicEntityInfo.canGetRole(entity, bankId).toString, mayReadField(entity, field)))
+ byRole.toList ++ fromPublic.toList
+ }
+ val refusal = accessProblem(bankId, declaration, plan, mayReadEntity, mayReadField)
+ .orElse(if (parentDefinition.useRowLevelAccess && plan.joins.nonEmpty || !pageReadable(bankId, from, plan) && plan.joins.nonEmpty)
+ Some(DynamicQueryFailure(400, DynamicEntityJoinRequiresProjection)) else None)
+ DynamicQueryExplanation(space, callerUserId.isEmpty, refusal.isEmpty, refusal.map(_.message), entities, restricted,
+ rules(space), pageSteps(bankId, declaration, parentDefinition, plan, callerUserId) ++ joinSteps(bankId, from, joins))
+ }
+ }
+
+ /** True when the page would be read from the projection (see [[readPage]], which makes the same choice). */
+ private def pageReadable(bankId: Option[String], from: String, plan: QueryPlan): Boolean =
+ IndexingCapabilities.projectionEnabled && (ProjectionReadiness.planFields(plan).nonEmpty || plan.joins.nonEmpty) &&
+ ProjectionReadiness.ready(bankId, from, plan)
+
+ private def rules(space: String): List[String] = List(
+ s"Only Dynamic Entities of space $space are read, and only through their definitions: no other OBP data can be named.",
+ "Only shared records are used, never a User's personal records, whoever owns them.",
+ "For an entity with row-level access, only the records the caller's access list allows are used.",
+ "A field that requires a read Role is null (or left out, when not selected) unless the caller holds that Role, and the caller cannot filter or sort on it.",
+ "A joined value is null when there is no matching record, when the caller may not read it, or when it lacks the field, so a join never reveals a hidden record.",
+ "Every value is bound as a parameter (shown as ?); table and column names come from the definitions.")
+
+ private def pageSteps(bankId: Option[String], declaration: DynamicQueryDeclaration, parent: DynamicEntityInfo,
+ plan: QueryPlan, callerUserId: Option[String]): List[ExplainedStep] = {
+ val from = declaration.from
+ val wantsCount = declaration.envelope.count.isDefined
+ if (parent.useRowLevelAccess)
+ List(ExplainedStep(s"Read the page of '$from'", "record provider", None, List(
+ s"'$from' uses row-level access: its shared records are read through the record provider, only those the caller's access list allows are kept, and they are filtered, sorted and paged in memory.")))
+ else if (pageReadable(bankId, from, plan))
+ ExplainedStep(s"Read the page of '$from'", "projection",
+ PostgresProjectionBackend.sqlFor(from, bankId, callerUserId, isPersonalEntity = false, plan),
+ List("Filters and the sort run on the projection's indexed columns; only the records of the page are read.")) ::
+ (if (wantsCount) List(ExplainedStep(s"Count every match of '$from', for the envelope's '${declaration.envelope.count.getOrElse("")}'", "projection",
+ PostgresProjectionBackend.sqlFor(from, bankId, callerUserId, isPersonalEntity = false, plan, counting = true), Nil)) else Nil)
+ else {
+ val reason =
+ if (!IndexingCapabilities.projectionEnabled) "This instance does not use the query projection"
+ else if (ProjectionReadiness.planFields(plan).isEmpty && plan.joins.isEmpty) "The page neither filters nor sorts on an indexed field"
+ else "A field the page filters or sorts on is indexed but its projection column is not ready yet"
+ List(ExplainedStep(s"Read the page of '$from'", "record provider", None, List(
+ s"$reason, so every shared record of '$from' is read through the record provider and filtered, sorted and paged in memory." +
+ (if (wantsCount) " The count is the number of matches before paging." else ""))))
+ }
+ }
+
+ private def joinSteps(bankId: Option[String], from: String, joins: List[Join]): List[ExplainedStep] = {
+ val firstUse = scala.collection.mutable.Map[RecordJoiner.Link, Int]()
+ joins.zipWithIndex.map { case (join, index) =>
+ val number = index + 1
+ val link = RecordJoiner.linkOf(join)
+ val described = join.direction match {
+ case JoinDirection.Forward => s"Join $number follows '$from.${join.on}' to '${join.entity}' (forward)"
+ case JoinDirection.Reverse => s"Join $number: '${join.entity}' records whose '${join.on}' names the '$from' (reverse)"
+ }
+ val afterRead = List(
+ Some(s"Cardinality ${join.cardinality.name}" + join.order.map(o => s", ordered ${if (o.descending) "latest" else "earliest"} first by '${o.field}'").getOrElse("") + "."),
+ if (join.where.nonEmpty) Some(s"Its where filter is applied in memory to the records read: ${join.where.map(f => s"${f.field} ${f.op.name} ${f.values.mkString(",")}").mkString(", ")}.") else None,
+ if (definitionRowLevel(bankId, join.entity)) Some(s"'${join.entity}' uses row-level access: only records the caller's access list allows are used.") else None
+ ).flatten
+ firstUse.get(link) match {
+ case Some(earlier) =>
+ ExplainedStep(described, "shared", None, s"Uses the records already read for Join $earlier; nothing more is read." :: afterRead)
+ case None =>
+ firstUse(link) = number
+ join.direction match {
+ case JoinDirection.Forward =>
+ ExplainedStep(described, "record provider", None,
+ s"One read for the whole page: the '${join.entity}' records whose id is one of the page's '${join.on}' values; personal records are dropped." :: afterRead)
+ case JoinDirection.Reverse if IndexingCapabilities.projectionEnabled && ProjectionProvisioner.readyFields(bankId, join.entity).contains(join.on) =>
+ ExplainedStep(described, "projection",
+ Some(ProjectionStore.readByReferenceSql(ProjectionNaming.tableName(bankId, join.entity), ProjectionNaming.columnName(join.on),
+ bankId, join.entity, List("?"))),
+ s"One read for the whole page, through the index on '${join.on}': IN (...) holds one ? per record on the page, in batches of 1000." :: afterRead)
+ case JoinDirection.Reverse =>
+ ExplainedStep(described, "record provider", None,
+ s"The projection column for '${join.on}' is not in use here, so every shared record of '${join.entity}' is read and those whose '${join.on}' names a record of the page are kept." :: afterRead)
+ }
+ }
+ }
+ }
+
+ private def definitionRowLevel(bankId: Option[String], entityName: String): Boolean =
+ DynamicEntityHelper.definitionOf(bankId, entityName).exists(_.useRowLevelAccess)
+
+ /** Every entity a query reads for this plan: `from`, each join's entity, and each entity a caller's obp_exists names. */
+ private def entitiesRead(declaration: DynamicQueryDeclaration, plan: QueryPlan): List[String] =
+ (declaration.from :: declaration.joins.map(_.entity) ++ plan.joins.map(_.childEntity)).distinct
+
+ /**
+ * Why this caller may not run this query, or None. Shared by [[run]] and [[explain]], so the
+ * verdict an explanation shows is the one a call gets: first every entity the caller may not read
+ * (403, all of them named), then a filter or sort on a field the caller may not read (400).
+ */
+ private def accessProblem(bankId: Option[String], declaration: DynamicQueryDeclaration, plan: QueryPlan,
+ mayReadEntity: String => Boolean, mayReadField: (String, String) => Boolean): Option[DynamicQueryFailure] =
+ notReadable(bankId, entitiesRead(declaration, plan).filterNot(mayReadEntity)).orElse {
+ val from = declaration.from
+ (ProjectionReadiness.planFields(plan).filterNot(mayReadField(from, _)).map(from -> _) ++
+ plan.joins.flatMap(j => j.predicate.map(_.field).filterNot(mayReadField(j.childEntity, _)).map(j.childEntity -> _))).headOption
+ .map { case (entity, field) => invalid(s"You may not read '$field' on '$entity', so it cannot be filtered or sorted on.") }
+ }
+
+ /**
+ * The 403 for the entities the caller may not read, all of them at once, each with the Role that
+ * would let the caller read it and the bank it is needed at, so a caller missing several learns of
+ * them in one answer. None when the list is empty.
+ */
+ private def notReadable(bankId: Option[String], entities: List[String]): Option[DynamicQueryFailure] =
+ if (entities.isEmpty) None
+ else {
+ val bank = code.api.dynamic.entity.helper.DynamicEntitySpace.bankIdOrSystem(bankId)
+ val missing = entities.map(entity => s"$entity (needs ${DynamicEntityInfo.canGetRole(entity, bankId)} at bank $bank)")
+ Some(DynamicQueryFailure(403, s"$DynamicQueryEntityNotReadable${missing.mkString(", ")}."))
+ }
+
+ private def selectError(declaration: DynamicQueryDeclaration, parent: JoinEntityInfo): Option[DynamicQueryFailure] =
+ declaration.select.flatMap { fields =>
+ val declared = parent.propertyNames + parent.idFieldName
+ fields.find(f => !declared.contains(f)).map(f => invalid(s"'select' names '$f', which '${declaration.from}' does not have."))
+ .orElse(if (fields.distinct.size != fields.size) Some(invalid("'select' names a field twice.")) else None)
+ }
+
+ /** The page's plan: the declaration's `where` plus the caller's parameters, checked by the list endpoint's planner. */
+ private def planPage(bankId: Option[String], declaration: DynamicQueryDeclaration, parent: DynamicEntityInfo,
+ callerParams: Map[String, List[String]]): Either[DynamicQueryFailure, QueryPlan] =
+ (for {
+ parsed <- QueryParamParser.parse(callerParams)
+ (callerFilters, callerJoins, sort, page) = parsed
+ plan <- QueryPlanner.plan(declaration.where ++ callerFilters, callerJoins, sort, page, declaration.from,
+ parent.indexedFields, parent.referenceFields,
+ child => DynamicEntityHelper.definitionOf(bankId, child).map(i => JoinTargetInfo(i.indexedFields, i.referenceFields, i.unindexedReferenceFields)),
+ parent.unindexedReferenceFields)
+ } yield plan).left.map(e => invalid(e.message))
+
+ /** One page of shared parent records, and how many match in all (0 unless `wantTotal`). */
+ private def readPage(bankId: Option[String], from: String, parent: DynamicEntityInfo, plan: QueryPlan,
+ callerUserId: Option[String], wantTotal: Boolean): Either[DynamicQueryFailure, (List[JObject], Long)] = {
+ val fieldTypes = parent.indexedFields.map { case (name, spec) => name -> spec.fieldType }
+ def inMemory(records: List[JObject]): (List[JObject], Long) = {
+ val matching = InMemoryQueryExecutor.execute(records, plan.copy(page = Page.empty), fieldTypes)
+ val afterOffset = plan.page.offset.filter(_ > 0).fold(matching)(matching.drop)
+ (plan.page.limit.filter(_ >= 0).fold(afterOffset)(afterOffset.take), matching.size.toLong)
+ }
+ def parse(json: String): JObject = JsonAliases.parse(json).asInstanceOf[JObject]
+ val provider = DynamicDataProvider.connectorMethodProvider.vend
+
+ // The projection is used only when the plan filters, sorts or joins: an entity's projection table
+ // exists only once a field is indexed, and a plan touching no field gains nothing from it.
+ val projectionReady = pageReadable(bankId, from, plan)
+
+ if (parent.useRowLevelAccess) {
+ if (plan.joins.nonEmpty) Left(DynamicQueryFailure(400, DynamicEntityJoinRequiresProjection))
+ else {
+ val readable = callerUserId.map(DynamicDataAccessProvider.provider.vend.getReadableDynamicDataIds(bankId, from, _).toSet).getOrElse(Set.empty)
+ Right(inMemory(provider.getAllCommunity(bankId, from).filter(_.dynamicDataId.exists(readable.contains)).map(r => parse(r.dataJson))))
+ }
+ } else if (projectionReady) {
+ import cats.effect.unsafe.implicits.global
+ // The caller is passed so an obp_exists join onto a row-level entity counts only the rows the caller
+ // may read. (With isPersonalEntity = false it does not change which parent records are in scope.)
+ val page = PostgresProjectionBackend.query(from, bankId, callerUserId, isPersonalEntity = false, plan).unsafeRunSync()
+ val total = if (wantTotal) PostgresProjectionBackend.count(from, bankId, callerUserId, isPersonalEntity = false, plan).unsafeRunSync() else 0L
+ Right((page, total))
+ } else if (plan.joins.nonEmpty) {
+ // obp_exists / obp_not_exists are evaluated only by the projection.
+ Left(DynamicQueryFailure(400, DynamicEntityJoinRequiresProjection))
+ } else {
+ // Not enabled, or a field the plan needs is not ready yet: in memory gives the same answer, only slower.
+ Right(inMemory(provider.getAllDataJson(bankId, from, None, isPersonalEntity = false)))
+ }
+ }
+
+ /**
+ * The fields of one returned record: the selected ones in order (null when missing or not readable),
+ * or every field the caller may read; then the joined results, in the order of the joins.
+ */
+ private def project(record: JObject, select: Option[List[String]], joinedNames: List[String], parent: DynamicEntityInfo,
+ mayReadField: String => Boolean): JObject = {
+ val joinedFields = joinedNames.map(name => JField(name, record \ name))
+ select match {
+ case Some(fields) =>
+ JObject(fields.map(f => JField(f, if (mayReadField(f)) record \ f match { case JNothing => JNull; case v => v } else JNull)) ++ joinedFields)
+ case None =>
+ val restricted = (parent.readRestrictedFields ++ parent.publicHiddenFields).toSet.filterNot(mayReadField)
+ JObject(record.obj.filterNot { case (name, _) => restricted.contains(name) })
+ }
+ }
+}
diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/InMemoryQueryExecutor.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/InMemoryQueryExecutor.scala
index 03393349e6..adbe78ba56 100644
--- a/obp-api/src/main/scala/code/api/dynamic/entity/query/InMemoryQueryExecutor.scala
+++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/InMemoryQueryExecutor.scala
@@ -52,6 +52,17 @@ object InMemoryQueryExecutor {
paginate(sorted, plan.page)
}
+ /** True when the record satisfies every filter (AND), each judged by its field's declared type. */
+ def matchesAll(record: JObject, filters: List[Filter], fieldTypes: Map[String, DynamicEntityFieldType]): Boolean =
+ filters.forall(f => matches(f, record, typeOf(f.field, fieldTypes)))
+
+ /**
+ * Compare two field values of the given type: negative when `a` sorts first. A value that is present
+ * and of the declared type sorts before one that is missing or of another type, so comparing a
+ * value with JNothing says whether that value is usable.
+ */
+ def compareValues(fieldType: DynamicEntityFieldType, a: JValue, b: JValue): Int = cmp2(fieldType, a, b)
+
private def typeOf(field: String, fieldTypes: Map[String, DynamicEntityFieldType]): DynamicEntityFieldType =
fieldTypes.getOrElse(field, DynamicEntityFieldType.string)
diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/JoinPlanner.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/JoinPlanner.scala
new file mode 100644
index 0000000000..02af396d6a
--- /dev/null
+++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/JoinPlanner.scala
@@ -0,0 +1,392 @@
+/**
+Open Bank Project - API
+Copyright (C) 2011-2026, TESOBE GmbH.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+ */
+
+package code.api.dynamic.entity.query
+
+import code.api.dynamic.entity.helper.{DynamicEntityHelper, DynamicEntityInfo}
+import com.openbankproject.commons.model.enums.DynamicEntityFieldType
+import org.json4s.JsonAST.{JBool, JObject, JValue}
+
+/**
+ * A join adds to each record of a page something about the records of another entity that are linked
+ * to it by a `reference:` field. It is the join machinery Dynamic Queries are built on (see
+ * ideas/DYNAMIC_QUERIES.md); no endpoint accepts a join from a caller.
+ *
+ * A reference field links two entities, and it can be read from either end:
+ * - forward: the record's own field names the other record. An `activity` whose `operator_id` is
+ * `reference:operator` is joined to its operator. At most one record can match.
+ * - reverse: the other entity's field names this record. `certificate` records whose `activity_id` is
+ * `reference:activity` are joined to their activity. Any number can match.
+ *
+ * The author does not say which: the planner sees which entity holds the `on` field and infers it. Only
+ * when both readings are possible, as with a self-reference (`employee.manager_id` is
+ * `reference:employee`: my manager, or my direct reports?), must the author give `direction`.
+ *
+ * `cardinality` says what to do with the matching records: copy the fields of one (`at_most_one`), list
+ * them all (`many`), or say whether any exists (`exists`). A forward join is `at_most_one` unless it
+ * says `exists`, and can never be `many`. A reverse join must state its cardinality, and an
+ * `at_most_one` reverse join must say how to `pick` one when several match.
+ */
+sealed trait JoinDirection { def name: String }
+object JoinDirection {
+ case object Forward extends JoinDirection { val name = "forward" }
+ case object Reverse extends JoinDirection { val name = "reverse" }
+ val byName: Map[String, JoinDirection] = List(Forward, Reverse).map(d => d.name -> d).toMap
+}
+
+sealed trait Cardinality { def name: String }
+object Cardinality {
+ /** At most one matching record: its fields are copied to the top level of the record. */
+ case object AtMostOne extends Cardinality { val name = "at_most_one" }
+ /** Every matching record: an array of objects under one name. */
+ case object Many extends Cardinality { val name = "many" }
+ /** Whether a matching record exists: one value under one name. */
+ case object Exists extends Cardinality { val name = "exists" }
+
+ val all: List[Cardinality] = List(AtMostOne, Many, Exists)
+ val byName: Map[String, Cardinality] = all.map(c => c.name -> c).toMap
+}
+
+/** An order on one field of the matching records: `descending` puts the largest first. */
+case class RecordOrder(field: String, descending: Boolean)
+object RecordOrder {
+ /** `latest_by:` (largest first) or `earliest_by:` (smallest first); None for anything else. */
+ def parse(text: String): Option[RecordOrder] = text.split(":", 2) match {
+ case Array("latest_by", field) if field.trim.nonEmpty => Some(RecordOrder(field.trim, descending = true))
+ case Array("earliest_by", field) if field.trim.nonEmpty => Some(RecordOrder(field.trim, descending = false))
+ case _ => None
+ }
+}
+
+/**
+ * A join as an author writes it.
+ *
+ * - `entity`: the other entity; `on`: the `reference:` field that links the two, on either of them.
+ * - `direction`: `forward` or `reverse`; needed only when both are possible (see [[JoinDirection]]).
+ * - `cardinality`: `at_most_one`, `many` or `exists`. Optional for a forward join (`at_most_one`).
+ * - `where`: filters on the other entity's records, in the list endpoint's grammar (`eq`, `in`,
+ * `between`, ...), applied before the cardinality.
+ * - `fields`: (name in the result, field of the other record) pairs: for `at_most_one` they become
+ * fields of the record; for `many`, the fields of each element. Not used by `exists`.
+ * - `as`: the name of the result of `many` (the array) and `exists` (the value).
+ * - `pick`: for a reverse `at_most_one`, the rule that chooses one record when several match.
+ * - `order`: optional for `many`, the order of the array; by record id when absent.
+ * - `trueValue` / `falseValue`: for `exists`; JSON true and false when absent.
+ */
+case class JoinRequest(
+ entity: String,
+ on: String,
+ direction: Option[String] = None,
+ cardinality: Option[String] = None,
+ where: List[Filter] = Nil,
+ fields: List[(String, String)] = Nil,
+ as: Option[String] = None,
+ pick: Option[String] = None,
+ order: Option[String] = None,
+ trueValue: Option[JValue] = None,
+ falseValue: Option[JValue] = None
+)
+
+/** A join the planner has checked, with its direction decided. `order` is the parsed `pick` or `order`. */
+case class Join(
+ entity: String,
+ entityIdField: String,
+ on: String,
+ direction: JoinDirection,
+ cardinality: Cardinality,
+ where: List[Filter],
+ fields: List[(String, String)],
+ as: Option[String],
+ order: Option[RecordOrder],
+ trueValue: JValue,
+ falseValue: JValue,
+ entityFieldTypes: Map[String, DynamicEntityFieldType]
+) {
+ /** The names this join adds to the record. */
+ def resultNames: List[String] = cardinality match {
+ case Cardinality.AtMostOne => fields.map(_._1)
+ case _ => as.toList
+ }
+
+}
+
+/**
+ * This is what the join planner needs to know about one Dynamic Entity: the names of its declared
+ * properties, the name of the id field its records carry, every `reference:` field it
+ * declares (field name -> target entity, indexed or not), the declared types of its fields (for a
+ * `where` filter or an order), and which fields are declared `"indexed": true`.
+ */
+case class JoinEntityInfo(
+ propertyNames: Set[String],
+ idFieldName: String,
+ referenceTargets: Map[String, String],
+ fieldTypes: Map[String, DynamicEntityFieldType] = Map.empty,
+ indexedFieldNames: Set[String] = Set.empty
+)
+
+object JoinEntityInfo {
+ def of(info: DynamicEntityInfo): JoinEntityInfo =
+ JoinEntityInfo(info.propertyNames.toSet, info.idName, info.allReferenceFields, info.declaredFieldTypes, info.indexedFields.keySet)
+}
+
+/** A checked set of joins for one parent entity, ready to apply to pages of its records. */
+case class JoinPlan(parentEntityName: String, parentIdField: String, joins: List[Join]) {
+
+ /** Add every join's result to each record of the page, for the caller identified by `callerUserId` and `consumerId`. */
+ def apply(records: List[JObject], bankId: Option[String], callerUserId: Option[String], consumerId: String): List[JObject] =
+ RecordJoiner.join(records, joins, parentIdField, bankId, callerUserId, consumerId)
+}
+
+/**
+ * This object checks joins against the entity definitions before any record is read, so an author
+ * learns of a mistake from a clear message rather than from a null in every row.
+ *
+ * For each join, in order:
+ * 1. the other entity exists in the same space, and the caller may read it;
+ * 2. the direction: `on` must be a `reference:` field of the parent pointing at the other entity
+ * (forward), or of the other entity pointing at the parent (reverse); when both hold, `direction`
+ * must choose;
+ * 3. a reverse join's `on` field is declared `"indexed": true`, because the records that refer to a
+ * page of parents are looked up by it, which the query projection serves from an index (a forward
+ * join reads the other records by id, and needs no index);
+ * 4. every `where` filter names a declared field the caller may read, with an operator and value valid
+ * for its type (the same rules as the list endpoint's `obp_filter`);
+ * 5. the cardinality suits the direction, the result has the shape the cardinality needs (see
+ * [[JoinRequest]]), every copied field is declared, and a `pick` or `order` names a declared field
+ * the caller may read (ordering by a field the caller cannot see would reveal it);
+ * 6. every name it adds to the record is a usable field name, not taken by the parent's own fields
+ * or an earlier join.
+ */
+object JoinPlanner {
+
+ private val fieldNamePattern = "^[A-Za-z_][A-Za-z0-9_]{0,254}$".r
+
+ /**
+ * As [[plan]], reading the definitions from the stored Dynamic Entities of one space (`bankId`, None
+ * for the system space). `callerMayReadEntity` decides whether the caller may read another entity at
+ * all ([[DynamicEntityInfo.mayReadRecords]] is the rule a GET applies); field-level read restrictions
+ * are judged with [[DynamicEntityInfo.fieldReader]] for the caller (`callerUserId`, `consumerId`).
+ */
+ def planFor(
+ bankId: Option[String],
+ parentEntityName: String,
+ requests: List[JoinRequest],
+ callerUserId: Option[String],
+ consumerId: String,
+ callerMayReadEntity: String => Boolean
+ ): Either[QueryError, JoinPlan] = {
+ def infoOf(entityName: String): Option[JoinEntityInfo] =
+ DynamicEntityHelper.definitionOf(bankId, entityName).map(JoinEntityInfo.of)
+ for {
+ parent <- infoOf(parentEntityName).toRight(QueryError(s"There is no Dynamic Entity '$parentEntityName' in this space."))
+ joins <- plan(parentEntityName, parent, requests, infoOf, callerMayReadEntity,
+ DynamicEntityInfo.fieldReader(bankId, callerUserId, consumerId))
+ } yield JoinPlan(parentEntityName, parent.idFieldName, joins)
+ }
+
+ def plan(
+ parentEntityName: String,
+ parent: JoinEntityInfo,
+ requests: List[JoinRequest],
+ entityInfoOf: String => Option[JoinEntityInfo],
+ callerMayReadEntity: String => Boolean,
+ callerMayReadField: (String, String) => Boolean
+ ): Either[QueryError, List[Join]] = {
+ val parentFieldNames = parent.propertyNames + parent.idFieldName
+ requests.zipWithIndex.foldLeft(Right(Nil): Either[QueryError, List[Join]]) {
+ case (Left(error), _) => Left(error)
+ case (Right(planned), (request, index)) =>
+ val taken = parentFieldNames ++ planned.flatMap(_.resultNames)
+ planOne(parentEntityName, parent, taken, request, index, entityInfoOf, callerMayReadEntity, callerMayReadField)
+ .map(join => planned :+ join)
+ }
+ }
+
+ private def planOne(
+ parentEntityName: String,
+ parent: JoinEntityInfo,
+ taken: Set[String],
+ request: JoinRequest,
+ index: Int,
+ entityInfoOf: String => Option[JoinEntityInfo],
+ callerMayReadEntity: String => Boolean,
+ callerMayReadField: (String, String) => Boolean
+ ): Either[QueryError, Join] = {
+ val position = s"Join ${index + 1} ('${request.entity}' on '${request.on}')"
+ val entity = request.entity
+ for {
+ other <- entityInfoOf(entity).toRight(QueryError(s"$position: there is no Dynamic Entity '$entity' in this space."))
+ _ <- check(callerMayReadEntity(entity), s"$position: you may not read '$entity', so it cannot be joined to '$parentEntityName'.")
+ direction <- directionOf(position, parentEntityName, parent, entity, other, request)
+ _ <- check(direction == JoinDirection.Forward || other.indexedFieldNames.contains(request.on),
+ s"$position: '${request.on}' on '$entity' must be declared \"indexed\": true, because the '$entity' records of a page of " +
+ s"'$parentEntityName' are looked up by it. Add \"indexed\": true to that field and let the index build.")
+ _ <- firstError(request.where.map(filter => whereError(position, entity, other, filter, callerMayReadField)))
+ cardinality <- cardinalityOf(position, parentEntityName, entity, direction, request)
+ order <- shapeAndOrder(position, entity, other, direction, cardinality, request, callerMayReadField)
+ _ <- namesError(position, cardinality, request, taken)
+ } yield Join(entity, other.idFieldName, request.on, direction, cardinality, request.where, request.fields, request.as, order,
+ request.trueValue.getOrElse(JBool(true)), request.falseValue.getOrElse(JBool(false)), other.fieldTypes)
+ }
+
+ /** Which way `on` links the two entities: inferred, or checked when the author gave `direction`. */
+ private def directionOf(position: String, parentEntityName: String, parent: JoinEntityInfo, entity: String,
+ other: JoinEntityInfo, request: JoinRequest): Either[QueryError, JoinDirection] = {
+ val on = request.on
+ val forwardPossible = parent.referenceTargets.get(on).contains(entity)
+ val reversePossible = other.referenceTargets.get(on).contains(parentEntityName)
+ def forwardMissing = s"'$parentEntityName' has no field '$on' typed 'reference:$entity'"
+ def reverseMissing = s"'$entity' has no field '$on' typed 'reference:$parentEntityName'"
+ request.direction match {
+ case Some(text) =>
+ JoinDirection.byName.get(text).toRight(QueryError(s"$position: direction must be 'forward' or 'reverse'; got '$text'.")).flatMap {
+ case JoinDirection.Forward if !forwardPossible => Left(QueryError(s"$position: direction is forward, but $forwardMissing."))
+ case JoinDirection.Reverse if !reversePossible => Left(QueryError(s"$position: direction is reverse, but $reverseMissing."))
+ case chosen => Right(chosen)
+ }
+ case None =>
+ (forwardPossible, reversePossible) match {
+ case (true, false) => Right(JoinDirection.Forward)
+ case (false, true) => Right(JoinDirection.Reverse)
+ case (true, true) => Left(QueryError(
+ s"$position: '$on' links the two either way: forward (the '$parentEntityName' record's '$on' names a '$entity') or " +
+ s"reverse ('$entity' records whose '$on' names the '$parentEntityName'). Say which with \"direction\": \"forward\" or \"reverse\"."))
+ case (false, false) => Left(QueryError(
+ s"$position: '$on' must be a reference field linking '$parentEntityName' and '$entity', on either of them, but " +
+ s"$forwardMissing and $reverseMissing."))
+ }
+ }
+ }
+
+ private def cardinalityOf(position: String, parentEntityName: String, entity: String, direction: JoinDirection,
+ request: JoinRequest): Either[QueryError, Cardinality] = {
+ val stated: Either[QueryError, Option[Cardinality]] = request.cardinality match {
+ case None => Right(None)
+ case Some(text) => Cardinality.byName.get(text).map(Option(_)).toRight(QueryError(
+ s"$position: cardinality must be one of ${Cardinality.all.map(_.name).mkString(", ")}; got '$text'."))
+ }
+ stated.flatMap { cardinality =>
+ (direction, cardinality) match {
+ case (JoinDirection.Forward, Some(Cardinality.Many)) => Left(QueryError(
+ s"$position: this join follows '$parentEntityName.${request.on}', which names at most one '$entity', so it cannot be 'many'."))
+ case (JoinDirection.Forward, None) => Right(Cardinality.AtMostOne)
+ case (JoinDirection.Reverse, None) => Left(QueryError(
+ s"$position: this join finds the '$entity' records whose '${request.on}' names the '$parentEntityName', and there can be " +
+ s"several, so it needs \"cardinality\": ${Cardinality.all.map(_.name).mkString(", ")}."))
+ case (_, Some(chosen)) => Right(chosen)
+ }
+ }
+ }
+
+ private def whereError(position: String, entity: String, other: JoinEntityInfo, filter: Filter,
+ callerMayReadField: (String, String) => Boolean): Option[QueryError] =
+ if (!other.fieldTypes.contains(filter.field))
+ Some(QueryError(s"$position: the where filter names '${filter.field}', which '$entity' does not declare with a type that can be filtered."))
+ else if (!callerMayReadField(entity, filter.field))
+ Some(QueryError(s"$position: you may not read '${filter.field}' on '$entity', so it cannot be filtered on."))
+ else {
+ val scalarSpecs = other.fieldTypes.map { case (name, fieldType) => name -> FieldSpec(fieldType, OperatorMatrix.SCALAR) }
+ QueryPlanner.validateFilter(filter, scalarSpecs).map(e => QueryError(s"$position: ${e.message}"))
+ }
+
+ private def shapeAndOrder(position: String, entity: String, other: JoinEntityInfo, direction: JoinDirection,
+ cardinality: Cardinality, request: JoinRequest,
+ callerMayReadField: (String, String) => Boolean): Either[QueryError, Option[RecordOrder]] = {
+ val copyable = other.propertyNames + other.idFieldName
+ def copiedFieldsDeclared: Either[QueryError, Unit] =
+ request.fields.map(_._2).find(f => !copyable.contains(f))
+ .map(f => QueryError(s"$position: '$entity' has no field '$f'.")).toLeft(())
+ def parsedOrder(text: String, what: String): Either[QueryError, RecordOrder] =
+ for {
+ order <- RecordOrder.parse(text).toRight(QueryError(
+ s"$position: $what must be 'latest_by:' or 'earliest_by:'; got '$text'."))
+ _ <- check(other.fieldTypes.contains(order.field), s"$position: $what names '${order.field}', which '$entity' does not declare with a type that can be ordered.")
+ _ <- check(callerMayReadField(entity, order.field), s"$position: you may not read '${order.field}' on '$entity', so it cannot be ordered by.")
+ } yield order
+ def absent(value: Option[_], name: String, reason: String): Either[QueryError, Unit] =
+ check(value.isEmpty, s"$position: '$name' is not used $reason.")
+ val withCardinality = s"with cardinality ${cardinality.name}"
+ val oneAtMost = "by a join that can find at most one record"
+
+ cardinality match {
+ case Cardinality.AtMostOne =>
+ for {
+ _ <- check(request.fields.nonEmpty, s"$position: at_most_one needs 'fields', the fields to copy from the matching record.")
+ _ <- absent(request.as, "as", withCardinality)
+ _ <- absent(request.order, "order", withCardinality)
+ _ <- absent(request.trueValue.orElse(request.falseValue), "true_value / false_value", withCardinality)
+ _ <- copiedFieldsDeclared
+ order <- direction match {
+ case JoinDirection.Forward => absent(request.pick, "pick", oneAtMost).map(_ => None)
+ case JoinDirection.Reverse =>
+ request.pick.toRight(QueryError(
+ s"$position: at_most_one needs 'pick' ('latest_by:' or 'earliest_by:'), the rule that chooses one record " +
+ s"when several '$entity' records name the same record in '${request.on}'."))
+ .flatMap(parsedOrder(_, "pick")).map(Option(_))
+ }
+ } yield order
+ case Cardinality.Many =>
+ for {
+ _ <- check(request.fields.nonEmpty, s"$position: many needs 'fields', the fields of each element of the array.")
+ _ <- absent(request.pick, "pick", withCardinality)
+ _ <- absent(request.trueValue.orElse(request.falseValue), "true_value / false_value", withCardinality)
+ _ <- copiedFieldsDeclared
+ elementNames = request.fields.map(_._1)
+ _ <- elementNames.find(n => !fieldNamePattern.matches(n))
+ .map(n => QueryError(s"$position: element field name '$n' must be made of letters, digits and underscores, not starting with a digit.")).toLeft(())
+ _ <- check(elementNames.distinct.size == elementNames.size, s"$position: an element field name is used twice.")
+ order <- request.order.map(parsedOrder(_, "order").map(Option(_))).getOrElse(Right(None))
+ } yield order
+ case Cardinality.Exists =>
+ for {
+ _ <- check(request.fields.isEmpty, s"$position: 'fields' is not used $withCardinality.")
+ _ <- absent(request.pick, "pick", withCardinality)
+ _ <- absent(request.order, "order", withCardinality)
+ } yield None
+ }
+ }
+
+ private def namesError(position: String, cardinality: Cardinality, request: JoinRequest, taken: Set[String]): Either[QueryError, Unit] = {
+ val added: Either[QueryError, List[String]] = cardinality match {
+ case Cardinality.AtMostOne => Right(request.fields.map(_._1))
+ case _ => request.as.toRight(QueryError(s"$position: ${cardinality.name} needs 'as', the name of its result.")).map(List(_))
+ }
+ added.flatMap { names =>
+ names.find(n => !fieldNamePattern.matches(n))
+ .map(n => QueryError(s"$position: '$n' must be a field name made of letters, digits and underscores, not starting with a digit."))
+ .orElse(names.find(taken.contains).map(n => QueryError(s"$position: '$n' is already a field of the result. Choose another name.")))
+ .orElse(if (names.distinct.size != names.size) Some(QueryError(s"$position: a result name is used twice.")) else None)
+ .toLeft(())
+ }
+ }
+
+ private def check(condition: Boolean, message: => String): Either[QueryError, Unit] =
+ if (condition) Right(()) else Left(QueryError(message))
+
+ private def firstError(results: List[Option[QueryError]]): Either[QueryError, Unit] =
+ results.flatten.headOption.toLeft(())
+}
diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryParamParser.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryParamParser.scala
index e0a09c7e67..028755cd86 100644
--- a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryParamParser.scala
+++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryParamParser.scala
@@ -115,7 +115,7 @@ object QueryParamParser {
sequence(perKey).map(_.flatten)
}
- private def parseOneFilter(field: String, raw: String): Either[QueryError, Filter] = {
+ private[query] def parseOneFilter(field: String, raw: String): Either[QueryError, Filter] = {
val idx = raw.indexOf(':')
if (idx < 0)
// No ':' — only a nullary operator (is_null / not_set) is valid in this form.
diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryPlanner.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryPlanner.scala
index 78c6a5ddb1..2e71f0e742 100644
--- a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryPlanner.scala
+++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryPlanner.scala
@@ -180,7 +180,7 @@ object QueryPlanner {
// ----- per-term validation -----
- private def validateFilter(f: Filter, indexedFields: Map[String, FieldSpec]): Option[QueryError] =
+ private[query] def validateFilter(f: Filter, indexedFields: Map[String, FieldSpec]): Option[QueryError] =
indexedFields.get(f.field) match {
case None => Some(QueryError(s"Field '${f.field}' is not queryable (it is not declared indexed)."))
case Some(spec) =>
diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/RecordJoiner.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/RecordJoiner.scala
new file mode 100644
index 0000000000..e07085d0ea
--- /dev/null
+++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/RecordJoiner.scala
@@ -0,0 +1,205 @@
+/**
+Open Bank Project - API
+Copyright (C) 2011-2026, TESOBE GmbH.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+ */
+
+package code.api.dynamic.entity.query
+
+import cats.effect.unsafe.implicits.{global => ioRuntime}
+import code.DynamicData.{DynamicDataAccessProvider, DynamicDataProvider}
+import code.api.dynamic.entity.helper.{DynamicEntityHelper, DynamicEntityInfo}
+import code.api.dynamic.entity.projection.{IndexingCapabilities, ProjectionDb, ProjectionNaming, ProjectionProvisioner, ProjectionStore}
+import com.openbankproject.commons.util.JsonAliases
+import org.json4s.JsonAST._
+
+/**
+ * This object applies planned [[Join]]s to a page of records.
+ *
+ * It runs after the page has been filtered, sorted and paginated, and never adds, removes or reorders
+ * a record of the page, so it cannot change which records a query returns.
+ *
+ * The other records are read in one batch per (entity, `on` field, direction) for the whole page:
+ * - forward: by id (the provider's `getByIds`), on any storage backend, so the `on` field needs no
+ * index;
+ * - reverse: through the entity's query projection when it is enabled and the `on` column is ready,
+ * so the lookup uses its index; otherwise by reading the entity's shared records and selecting in
+ * memory, which is correct everywhere, only slower.
+ * `where` and the ordering are evaluated in memory on the records read, so they may use fields that
+ * are not indexed. Then both directions are merged the same way: the matching records, filtered by
+ * `where`, give one record's fields, a list, or a yes/no.
+ *
+ * What a caller can see through a join is no more than what it could read directly:
+ * - only shared records of the other entity are used, never a User's personal record, whoever owns it;
+ * - for an entity with row-level access, only the records the caller's access list lets them read;
+ * - a field declared `read_role_required` is copied only for a caller holding its read Role, and a field
+ * declared `hide_field_from_public_access` only for a caller who reaches the entity other than
+ * through its public access.
+ * Whether the caller may read the other entity at all is checked by [[JoinPlanner]]. A copied value is
+ * JSON null when there is no matching record, when it may not be read, when it lacks the field, or when
+ * the field may not be read; these are deliberately indistinguishable, so a join never reveals that a
+ * record the caller cannot read exists.
+ */
+object RecordJoiner {
+
+ /** The key a batch of other records is read under. */
+ case class Link(entity: String, on: String, direction: JoinDirection)
+
+ /** Other records by link, then by the parent id they belong to. */
+ type LinkedRecords = Map[Link, Map[String, List[JObject]]]
+
+ def join(records: List[JObject], joins: List[Join], parentIdField: String,
+ bankId: Option[String], callerUserId: Option[String], consumerId: String): List[JObject] =
+ if (joins.isEmpty || records.isEmpty) records
+ else {
+ val links = joins.map(linkOf).distinct
+ val linked: LinkedRecords = links.map { link =>
+ link -> (link.direction match {
+ case JoinDirection.Forward => fetchForward(bankId, link, records, callerUserId)
+ case JoinDirection.Reverse => fetchReverse(bankId, link, records.flatMap(idOf(_, parentIdField)).distinct, callerUserId)
+ })
+ }.toMap
+ merge(records, joins, parentIdField, linked, DynamicEntityInfo.fieldReader(bankId, callerUserId, consumerId))
+ }
+
+ def linkOf(join: Join): Link = Link(join.entity, join.on, join.direction)
+
+ /**
+ * This adds each join's result to each record, from other records already read. It reads nothing, so
+ * it is the part to test without a database. `fieldReadable(entity, field)` says whether the caller
+ * may see that field.
+ *
+ * The other records belong to a parent record by the parent's id: for a forward join that is the id
+ * its `on` field names, held in the parent; for a reverse join, the parent's own id.
+ */
+ def merge(records: List[JObject], joins: List[Join], parentIdField: String, linked: LinkedRecords,
+ fieldReadable: (String, String) => Boolean): List[JObject] = {
+ val readable: Map[(String, String), Boolean] =
+ joins.flatMap(j => j.fields.map(f => (j.entity, f._2))).distinct.map(key => key -> fieldReadable(key._1, key._2)).toMap
+ def copy(join: Join, from: JObject, field: String): JValue =
+ if (!readable((join.entity, field))) JNull
+ else from \ field match { case JNothing => JNull; case value => value }
+
+ records.map { record =>
+ val added = joins.flatMap { join =>
+ val key = join.direction match {
+ case JoinDirection.Forward => idOf(record, join.on)
+ case JoinDirection.Reverse => idOf(record, parentIdField)
+ }
+ val matching = key.flatMap(k => linked.get(linkOf(join)).flatMap(_.get(k))).getOrElse(Nil)
+ .filter(other => InMemoryQueryExecutor.matchesAll(other, join.where, join.entityFieldTypes))
+ join.cardinality match {
+ case Cardinality.Exists =>
+ List(JField(join.as.getOrElse(""), if (matching.nonEmpty) join.trueValue else join.falseValue))
+ case Cardinality.AtMostOne =>
+ val chosen = ordered(matching, join).headOption
+ join.fields.map { case (name, field) => JField(name, chosen.map(copy(join, _, field)).getOrElse(JNull)) }
+ case Cardinality.Many =>
+ val elements = ordered(matching, join).map(other =>
+ JObject(join.fields.map { case (name, field) => JField(name, copy(join, other, field)) }))
+ List(JField(join.as.getOrElse(""), JArray(elements)))
+ }
+ }
+ JObject(record.obj ++ added)
+ }
+ }
+
+ /**
+ * Matching records in the join's order, or by record id when it has none. A record whose order field
+ * is missing, or not of its declared type, comes after every record that has one, whichever
+ * direction; ties are broken by record id, so the result never depends on the order the records were
+ * read in.
+ */
+ private def ordered(records: List[JObject], join: Join): List[JObject] = {
+ def id(record: JObject): String = idOf(record, join.entityIdField).getOrElse("")
+ join.order match {
+ case None => records.sortBy(id)
+ case Some(order) =>
+ val fieldType = join.entityFieldTypes(order.field)
+ def present(record: JObject): Boolean = InMemoryQueryExecutor.compareValues(fieldType, record \ order.field, JNothing) < 0
+ def compare(a: JObject, b: JObject): Int = (present(a), present(b)) match {
+ case (true, true) =>
+ val byValue = InMemoryQueryExecutor.compareValues(fieldType, a \ order.field, b \ order.field)
+ val directed = if (order.descending) -byValue else byValue
+ if (directed != 0) directed else id(a).compareTo(id(b))
+ case (true, false) => -1
+ case (false, true) => 1
+ case (false, false) => id(a).compareTo(id(b))
+ }
+ records.sortWith((a, b) => compare(a, b) < 0)
+ }
+ }
+
+ /** The records the page's `on` fields name, readable by the caller, by their own id. */
+ def fetchForward(bankId: Option[String], link: Link, records: List[JObject], callerUserId: Option[String]): Map[String, List[JObject]] = {
+ val ids = records.flatMap(idOf(_, link.on)).distinct
+ if (ids.isEmpty) Map.empty
+ else {
+ val readable = rowAccess(bankId, link.entity, callerUserId)
+ DynamicDataProvider.connectorMethodProvider.vend.getByIds(bankId, link.entity, ids)
+ .filterNot(_.isPersonalEntity)
+ .flatMap(row => row.dynamicDataId.filter(readable).map(id => id -> List(JsonAliases.parse(row.dataJson).asInstanceOf[JObject])))
+ .toMap
+ }
+ }
+
+ /** The records whose `on` field names one of `parentIds`, readable by the caller, by that parent id. */
+ def fetchReverse(bankId: Option[String], link: Link, parentIds: List[String], callerUserId: Option[String]): Map[String, List[JObject]] =
+ if (parentIds.isEmpty) Map.empty
+ else {
+ val readable = rowAccess(bankId, link.entity, callerUserId)
+ val rows: List[(String, JObject)] =
+ if (IndexingCapabilities.projectionEnabled && ProjectionProvisioner.readyFields(bankId, link.entity).contains(link.on))
+ parentIds.grouped(1000).toList.flatMap { someIds =>
+ ProjectionDb.run(ProjectionStore.readByReference(ProjectionNaming.tableName(bankId, link.entity), ProjectionNaming.columnName(link.on),
+ bankId, link.entity, someIds)).unsafeRunSync()(ioRuntime)
+ }.map { case (id, json) => id -> JsonAliases.parse(json).asInstanceOf[JObject] }
+ else {
+ val wanted = parentIds.toSet
+ DynamicDataProvider.connectorMethodProvider.vend.getAll(bankId, link.entity, None, isPersonalEntity = false)
+ .flatMap(row => row.dynamicDataId.map(_ -> JsonAliases.parse(row.dataJson).asInstanceOf[JObject]))
+ .filter { case (_, other) => idOf(other, link.on).exists(wanted.contains) }
+ }
+ rows.collect { case (id, other) if readable(id) => other }.groupBy(other => idOf(other, link.on).getOrElse(""))
+ }
+
+ /**
+ * This says, record id by record id, whether the caller may read a record of `entityName` that is
+ * otherwise in scope. For an entity without row-level access every record is allowed; for one with
+ * it, only the records the caller's access list lets them read, and none for an anonymous caller.
+ * The access list is read once, when this is called.
+ */
+ private def rowAccess(bankId: Option[String], entityName: String, callerUserId: Option[String]): String => Boolean =
+ if (!DynamicEntityHelper.definitionOf(bankId, entityName).exists(_.useRowLevelAccess)) _ => true
+ else {
+ val allowed = callerUserId.map(DynamicDataAccessProvider.provider.vend.getReadableDynamicDataIds(bankId, entityName, _).toSet).getOrElse(Set.empty)
+ allowed.contains
+ }
+
+ private def idOf(record: JObject, field: String): Option[String] =
+ record \ field match {
+ case JString(id) if id.trim.nonEmpty => Some(id)
+ case _ => None
+ }
+}
diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala
index a281069569..a8c6ab2ead 100644
--- a/obp-api/src/main/scala/code/api/util/APIUtil.scala
+++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala
@@ -5042,6 +5042,20 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
def allDynamicResourceDocs= (DynamicEntityHelper.doc ++ DynamicEndpointHelper.doc ++ DynamicEndpoints.dynamicResourceDocs).toList
+ /**
+ * This function says whether a dynamic ResourceDoc belongs to one space, which is what the bank
+ * level resource-docs endpoints (/banks/BANK_ID/resource-docs/...) list.
+ *
+ * A space is a bank id, or Constant.DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID ("SYS") for the system
+ * space. Dynamic docs record the system space as belonging to no bank at all (createdByBankId is
+ * None), so SYS matches those docs as well as any doc that names SYS explicitly.
+ */
+ def dynamicResourceDocBelongsToSpace(doc: ResourceDoc, space: String): Boolean =
+ doc.createdByBankId.filter(_.nonEmpty) match {
+ case None => space == DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID
+ case Some(bankId) => bankId == space
+ }
+
/**
* The dynamic docs a versioned resource-docs listing shows. v7.0.0 documents Dynamic Entity records at
* their v7.0.0 URLs (/obp/v7.0.0/banks/BANK_ID/dynamic-entities/...); every other version at the
diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala
index fac7b40ff4..f7073b3452 100644
--- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala
+++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala
@@ -108,7 +108,8 @@ object ErrorMessages {
val RowLevelAccessRequiresLocalBacking = "OBP-09020: use_row_level_access is only supported for locally-backed dynamic entities. This entity is routed to an external connector (a method routing for dynamicEntityProcess exists for it), where the row-level ACL cannot be enforced. Remove the method routing or disable use_row_level_access."
val RowLevelAccessNotEnabled = "OBP-09021: The row-access endpoints are only available for dynamic entities created with use_row_level_access = true."
val DynamicEntityJoinRequiresProjection = "OBP-09022: obp_exists / obp_not_exists join queries require the SQL projection backend (dynamic_entity.indexing.backend=auto on a supported database). This deployment serves Dynamic Entity reads in-memory, where joins are not supported."
- val DynamicEntityUpdateNotSchemaCompatible = "OBP-09023: Operation is not allowed, because this DynamicEntity already has data. The definition of a populated entity can only be changed in schema-compatible ways: the entity name, the set of property names and each property's type must stay the same, and no property may be added to 'required'. Changing indexed, index, example, description, minLength, maxLength and the read/write role settings is allowed. Delete all the data before making a structural change."
+ val DynamicEntityUpdateNotSchemaCompatible = "OBP-09023: Operation is not allowed, because this DynamicEntity already has data. The definition of a populated entity can only be changed in schema-compatible ways: the entity name must stay the same, every existing property must keep its name and type, and no property may be added to 'required'. New optional properties may be added. Changing indexed, index, example, description, minLength, maxLength, the read/write role settings and hide_field_from_public_access is allowed. Delete all the data before making a structural change."
+ val DynamicEntityFieldNotReadable = "OBP-09025: These fields cannot be used to filter or sort, because you may not read them: "
val DynamicEntityRecordIdTooLong = "OBP-09024: The id of this DynamicEntity record is too long. A record id is stored in a column of 255 characters. Please supply a shorter id, or leave the id field out of the request body and one will be generated."
@@ -1017,6 +1018,8 @@ object ErrorMessages {
"A payment started on somebody else's behalf is authorised by that person, not by the caller that started it."
val PaymentChallengeHasNoOnBehalfOfUser = "OBP-40064: This payment needs Strong Customer Authentication, but the user it is being made for could not be determined, " +
"so there is nobody who can be asked to authorise it. A consent that names the user it acts for is required before a payment of this size can be started."
+ val DynamicQueryInvalid = "OBP-40065: The Dynamic Query cannot be run as written. "
+ val DynamicQueryEntityNotReadable = "OBP-40066: This Dynamic Query reads Dynamic Entities you may not read: "
// Exceptions (OBP-50XXX)
val UnknownError = "OBP-50000: Unknown Error."
val FutureTimeoutException = "OBP-50001: Future Timeout Exception."
diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala
index 9a2bd7f4ab..f91947c1f7 100644
--- a/obp-api/src/main/scala/code/api/util/Glossary.scala
+++ b/obp-api/src/main/scala/code/api/util/Glossary.scala
@@ -648,6 +648,41 @@ object Glossary extends MdcLoggable {
|```
|(Default: 1000 requests per hour. `0` blocks all anonymous access, `-1` removes the limit.)
|
+ |### Three rate limiters
+ |
+ |OBP runs three independent rate limiters. They are checked in this order, and each answers **429** with its own error code so a client can tell which counter it hit:
+ |
+ |1. **Self-service limiter** (`self_service.rate_limit.*`) runs first, before routing and before any authentication, keyed by the client IP address. It covers the endpoints anyone can call before the bank has granted them anything. Trip code: `OBP-10060`.
+ |2. **Authentication limiter** (`auth.rate_limit.*`) runs inside the credential check of Direct Login, DAuth, Gateway Login and SIWE, before the password or token is verified, keyed by IP address and by account. It defends against brute force, credential stuffing and lockout attacks. Trip code: `OBP-10061`.
+ |3. **Consumer quota** (the limits described above) runs after authentication, keyed by Consumer, or by IP address with a single hourly ceiling for anonymous calls. It is the commercial and fair-use quota. Trip code: `OBP-10018`.
+ |
+ |Before all three, an operator can put a single IP address under a temporary **IP penalty**: a per-minute limit on every endpoint, for a set time, for example during a scan or denial-of-service attempt (`POST /obp/v7.0.0/management/ip-penalties`, Role CanCreateIpPenalty). A per-minute limit of 0 refuses every request. Penalties are always enforced, shared by every instance, and disappear when they expire; the penalty endpoints themselves are never refused, so a mistake can be undone. Trip code: `OBP-10062`.
+ |
+ |A login attempt is counted by the authentication limiter only; it is not a self-service scope, so no attempt is counted twice. Every limiter counts in Redis and fails open: a Redis outage never blocks a call.
+ |
+ |### Self-service rate limiting (per IP address, before any credential)
+ |
+ |The limits above are keyed by Consumer, so they cannot protect the calls a client makes before it has one. Those endpoints are covered by the self-service limiter, keyed by the client IP address, grouped in scopes:
+ |
+ |- **signup** — Create User (self-registration), Validate User Email, Get User Invitation Information
+ |- **password_reset** — Request Password Reset Email, Complete Password Reset
+ |- **consent_request** — Create Consent Request, Create Consent Request VRP
+ |- **consumer_registration** — Create a Consumer (Dynamic Registration)
+ |- **lookup** — Validate and check IBAN
+ |- **signal_channel_create** — Publish Signal Message, counted only when it creates a new channel, over REST and gRPC alike (gRPC uses the socket peer address; if none is available it falls back to the Consumer)
+ |
+ |Each scope has per-minute, per-hour and per-day limits per IP, with built-in defaults chosen so that a person or a well-behaved agent never reaches them, plus an optional global per-hour cap across all addresses that acts as a circuit breaker. Every request is counted, whether or not it succeeds. Counters live in Redis and fail open.
+ |
+ |**Shadow mode (the default).** The limiter is on out of the box but does not block. A request over a limit is logged once per window (`event=self_service_rate_limit_shadow_trip`) and the response carries:
+ |
+ | X-Rate-Limit-Warning: OBP-10059: Could conflict with a Future Rate Limit: This request might exceed the rate limit for signup (5 per hour) in the future.
+ |
+ |No enforcement date is claimed unless the operator sets `self_service.rate_limit.enforce_announced_from`, in which case ", from " is appended. Every self-service response also carries `X-Rate-Limit-Limit`, `X-Rate-Limit-Remaining` and `X-Rate-Limit-Reset` for the window the caller is closest to exhausting, so a client can back off before enforcement starts.
+ |
+ |**Enforce mode.** Set `self_service.rate_limit.mode = enforce` and a trip answers **429** with `OBP-10060`, a `Retry-After` header and the same `X-Rate-Limit-*` headers, without running the endpoint.
+ |
+ |Limits are set with `self_service.rate_limit..per_ip.per_minute|per_hour|per_day`, `self_service.rate_limit..global.per_hour`, or the generic `self_service.rate_limit.per_ip.*`; -1 switches a window off and 0 blocks it. See the props template for the built-in numbers. Behind a proxy, configure `trust.proxy.enabled` and `trust.proxy.header` so the client address is the real one; otherwise every caller shares the proxy's counters.
+ |
|### Related Concepts
|
|- **Consumer**: The API client subject to rate limiting
@@ -786,41 +821,6 @@ object Glossary extends MdcLoggable {
|
|This glossary item is Work In Progress.
|
- |
- |### Three rate limiters
- |
- |OBP runs three independent rate limiters. They are checked in this order, and each answers **429** with its own error code so a client can tell which counter it hit:
- |
- |1. **Self-service limiter** (`self_service.rate_limit.*`) runs first, before routing and before any authentication, keyed by the client IP address. It covers the endpoints anyone can call before the bank has granted them anything. Trip code: `OBP-10060`.
- |2. **Authentication limiter** (`auth.rate_limit.*`) runs inside the credential check of Direct Login, DAuth, Gateway Login and SIWE, before the password or token is verified, keyed by IP address and by account. It defends against brute force, credential stuffing and lockout attacks. Trip code: `OBP-10061`.
- |3. **Consumer quota** (the limits described above) runs after authentication, keyed by Consumer, or by IP address with a single hourly ceiling for anonymous calls. It is the commercial and fair-use quota. Trip code: `OBP-10018`.
- |
- |Before all three, an operator can put a single IP address under a temporary **IP penalty**: a per-minute limit on every endpoint, for a set time, for example during a scan or denial-of-service attempt (`POST /obp/v7.0.0/management/ip-penalties`, Role CanCreateIpPenalty). A per-minute limit of 0 refuses every request. Penalties are always enforced, shared by every instance, and disappear when they expire; the penalty endpoints themselves are never refused, so a mistake can be undone. Trip code: `OBP-10062`.
- |
- |A login attempt is counted by the authentication limiter only; it is not a self-service scope, so no attempt is counted twice. Every limiter counts in Redis and fails open: a Redis outage never blocks a call.
- |
- |### Self-service rate limiting (per IP address, before any credential)
- |
- |The limits above are keyed by Consumer, so they cannot protect the calls a client makes before it has one. Those endpoints are covered by the self-service limiter, keyed by the client IP address, grouped in scopes:
- |
- |- **signup** — Create User (self-registration), Validate User Email, Get User Invitation Information
- |- **password_reset** — Request Password Reset Email, Complete Password Reset
- |- **consent_request** — Create Consent Request, Create Consent Request VRP
- |- **consumer_registration** — Create a Consumer (Dynamic Registration)
- |- **lookup** — Validate and check IBAN
- |- **signal_channel_create** — Publish Signal Message, counted only when it creates a new channel, over REST and gRPC alike (gRPC uses the socket peer address; if none is available it falls back to the Consumer)
- |
- |Each scope has per-minute, per-hour and per-day limits per IP, with built-in defaults chosen so that a person or a well-behaved agent never reaches them, plus an optional global per-hour cap across all addresses that acts as a circuit breaker. Every request is counted, whether or not it succeeds. Counters live in Redis and fail open.
- |
- |**Shadow mode (the default).** The limiter is on out of the box but does not block. A request over a limit is logged once per window (`event=self_service_rate_limit_shadow_trip`) and the response carries:
- |
- | X-Rate-Limit-Warning: OBP-10059: Could conflict with a Future Rate Limit: This request might exceed the rate limit for signup (5 per hour) in the future.
- |
- |No enforcement date is claimed unless the operator sets `self_service.rate_limit.enforce_announced_from`, in which case ", from " is appended. Every self-service response also carries `X-Rate-Limit-Limit`, `X-Rate-Limit-Remaining` and `X-Rate-Limit-Reset` for the window the caller is closest to exhausting, so a client can back off before enforcement starts.
- |
- |**Enforce mode.** Set `self_service.rate_limit.mode = enforce` and a trip answers **429** with `OBP-10060`, a `Retry-After` header and the same `X-Rate-Limit-*` headers, without running the endpoint.
- |
- |Limits are set with `self_service.rate_limit..per_ip.per_minute|per_hour|per_day`, `self_service.rate_limit..global.per_hour`, or the generic `self_service.rate_limit.per_ip.*`; -1 switches a window off and 0 blocks it. See the props template for the built-in numbers. Behind a proxy, configure `trust.proxy.enabled` and `trust.proxy.header` so the client address is the real one; otherwise every caller shares the proxy's counters.
""")
glossaryItems += GlossaryItem(
@@ -3627,7 +3627,17 @@ object Glossary extends MdcLoggable {
|
|**Supported field types:**
|
-|STRING, INTEGER, DOUBLE, BOOLEAN, DATE_WITH_DAY (format: yyyy-MM-dd), JSON (objects and arrays), and reference types (foreign keys)
+|Type names are case-sensitive:
+|
+|* `string`
+|* `integer` - whole numbers of any size. A value written with a decimal point, such as 1.0, is rejected.
+|* `number` - any number. Decimals are stored as a 64-bit binary floating-point value (about 15 to 17 significant digits), so values such as 0.1 are rounded slightly. Whole numbers keep full precision.
+|* `boolean` - true/false, or the strings "true"/"false"
+|* `DATE_WITH_DAY` - a string in the format yyyy-MM-dd
+|* `json` - a JSON object or array
+|* `reference:` - a foreign key to another entity
+|
+|For values that must be exact, such as money, use `integer` in minor units (for example, cents) or a `string`, not `number`.
|
|**The hasPersonalEntity flag:**
|
@@ -3703,6 +3713,9 @@ object Glossary extends MdcLoggable {
|
|* `write_role_required` (boolean) or `write_role` (explicit role name) — the field becomes **write-restricted**: it cannot be set via POST or PUT (its existing value is preserved), only via **PATCH** by a caller holding the field's write role.
|* `read_role_required` (boolean) or `read_role` (explicit role name) — the field becomes **read-restricted**: it is omitted from GET responses unless the caller holds the field's read role (public/anonymous access omits it entirely).
+|* `hide_field_from_public_access` (boolean) — for an entity with public access: the field is hidden from a caller whose access comes only from that public access (a caller who is not logged in, or one without the entity's read role), and shown to a caller holding the entity's read role, with no field role needed. The public endpoint always omits it.
+|
+|A caller can never filter or sort by a field they may not read, whether with a plain `?field=value` parameter, `obp_filter`, `obp_sort_by`, or the filter of an `obp_exists` join: whether a record came back would reveal the field's value. Such a request is refused with ${ErrorMessages.DynamicEntityFieldNotReadable.takeWhile(_ != ':')}, naming the fields.
|
|Restriction is on if either the boolean is `true` or an explicit role name is given. When a boolean is used, OBP auto-generates the role; e.g. for entity 'FooBar' field 'owner':
|
@@ -4138,7 +4151,9 @@ object Glossary extends MdcLoggable {
|
|Authentication and Role checks are applied to the compiled endpoint exactly as for Static endpoints - including the checks that run inside the shared authentication step: Consumer disabled, User locked / deleted, Consent processing and Rate Limiting.
|
-|Some cross-cutting features of the Static pipeline do *not* currently apply to runtime-compiled Dynamic Resource Doc endpoints: API Metrics are not recorded, the JSON Schema Validation and Force-Error interceptors are not run, the Idempotency-Key mechanism is unavailable, and handlers run on auto-commit (no request-scoped database transaction). Dynamic Endpoints created from Swagger (the proxy path) *do* record Metrics and *do* run the JSON Schema Validation interceptors.
+|Every call to a Dynamic Resource Doc endpoint is recorded as an API Metric, like a call to a Static endpoint: the response it gave, and also a call refused for missing authentication or Roles.
+|
+|Some other cross-cutting features of the Static pipeline do *not* currently apply to Dynamic Resource Doc endpoints: the JSON Schema Validation and Force-Error interceptors are not run, the Idempotency-Key mechanism is unavailable, and handlers run on auto-commit (no request-scoped database transaction). Dynamic Endpoints created from Swagger (the proxy path) *do* run the JSON Schema Validation interceptors.
|
|Because the method body is user-supplied code compiled at runtime, this feature is guarded by the `allow_user_generated_scala_code` prop (default: false) and the Roles CanCreateDynamicResourceDoc / CanCreateBankLevelDynamicResourceDoc etc.
|
@@ -4165,6 +4180,72 @@ object Glossary extends MdcLoggable {
|
|See ${getGlossaryItemLink("Dynamic Code Paths")} for how Dynamic Resource Docs relate to the other runtime-defined building blocks, and ${getGlossaryItemLink("Dynamic Change Request")} for how an operator can require a second person to approve each definition before it is compiled and served.
|
+|The method body is Scala unless `programming_lang` says otherwise: `Java` for a Java class, or `Query` for a declaration that reads Dynamic Entity records instead of code (see ${getGlossaryItemLink("Dynamic Query")}).
+|
+""".stripMargin)
+
+ glossaryItems += GlossaryItem(
+ title = "Dynamic Query",
+ description =
+ s"""
+|A Dynamic Query is a ${getGlossaryItemLink("Dynamic Resource Doc")} whose body is a declaration rather than code: its `programming_lang` is `Query`. It reads the records of one Dynamic Entity, adds the records joined to them through `reference:` fields, and returns them as a named list. It is for the common case of an Endpoint that only reads Dynamic Entity data, which would otherwise need a Scala method body.
+|
+|Because nothing is compiled or run, a Dynamic Query is available even where user-supplied code is switched off, and a reviewer approving one (see ${getGlossaryItemLink("Dynamic Change Request")}) reads a declaration, not a program. A Dynamic Query only reads, so its `request_verb` must be `GET`.
+|
+|**The body**
+|
+| {
+| "from": "activity",
+| "select": ["activity_id", "name", "city"],
+| "where": { "city": "eq:Berlin" },
+| "join": [
+| { "entity": "operator", "on": "operator_id",
+| "fields": { "operator_legal_name": "legal_name" } },
+| { "entity": "certificate", "on": "activity_id",
+| "cardinality": "at_most_one", "pick": "latest_by:issue_date",
+| "fields": { "certificate_number": "number" } },
+| { "entity": "inspection", "on": "activity_id",
+| "cardinality": "exists", "as": "inspected" }
+| ],
+| "envelope": { "rows": "activities", "count": "count" }
+| }
+|
+|* `from` (required): the Dynamic Entity whose records are returned.
+|* `select`: the fields of those records to return, in this order. All of them when absent.
+|* `where`: filters on them, written as the list Endpoint's `obp_filter` values (`"field": "eq:value"`, or a list of such strings for several filters on one field). Filtered fields must be declared `"indexed": true`.
+|* `join`: the related records to add to each record. See below.
+|* `envelope`: `rows` names the list (by default the entity's own list name) and `count`, when given, names a field holding how many records match in all, not only on this page.
+|
+|Keys that are not listed here are rejected, so a misspelt key is reported rather than ignored.
+|
+|**Joins**
+|
+|A `reference:` field links two entities, and a join can read it from either end. A *forward* join follows the record's own field to the record it names: an activity's `operator_id` names one operator. A *reverse* join finds the records of the other entity whose field names this record: certificates whose `activity_id` names the activity. The join only names the other `entity` and the field it is linked `on`; OBP sees which entity holds that field and works out the direction. Only for a self-reference, such as `employee.manager_id` of type `reference:employee` (the manager, or the direct reports?), must the join add `"direction": "forward"` or `"reverse"`.
+|
+|`cardinality` says what to do with the matching records:
+|
+|* `at_most_one` copies the fields of one record into the result (`fields` maps each result name to a field of that record). A forward join is `at_most_one` unless it says otherwise. A reverse join that is `at_most_one` must say how to choose when several records match, with `pick`: `latest_by:` or `earliest_by:`. A record without that field is never chosen ahead of one with it, and ties are broken by record id.
+|* `many` gives a list named `as`, one object per matching record with the `fields` given, ordered by `order` (`latest_by:` or `earliest_by:`), or by record id when absent. Only a reverse join can be `many`.
+|* `exists` gives one value named `as`: `true_value` if any record matches, `false_value` if none (JSON true and false by default).
+|
+|A join's `where` filters the other records before the cardinality applies, with the same operators as above. The field a reverse join is linked on must be declared `"indexed": true`.
+|
+|**Calling a Dynamic Query**
+|
+|A caller can narrow the result with the list Endpoint's own parameters: `obp_filter`, `obp_sort_by`, `obp_sort_direction`, `obp_limit`, `obp_offset`, `obp_exists` and `obp_not_exists`. They are added to the declaration's `where`, never replace it.
+|
+|A Dynamic Query can be public: callable without logging in. That needs all three of: no Roles on the Dynamic Resource Doc, an error list that does not name ${ErrorMessages.AuthenticatedUserIsRequired.takeWhile(_ != ':')} (which is how a Dynamic Resource Doc says it requires login), and public access on every Dynamic Entity it reads. Whether data can be public is decided on the entity; a query cannot make an entity's records public.
+|
+|**What a caller can see**
+|
+|A Dynamic Query never shows a caller more than they could read directly. The primary control is the Dynamic Entities' own access: the caller must be able to read every Dynamic Entity the query reads (its read Role, public access, or row-level access), exactly as through that entity's own Endpoints. Roles on the Dynamic Resource Doc itself are optional; they can only narrow who may call the query, never widen what a caller can read. Without that access the answer is ${ErrorMessages.DynamicQueryEntityNotReadable.takeWhile(_ != ':')} (403), naming every such entity with the Role that would let the caller read it and the bank it is needed at. Only shared records are used, never a User's personal records. For an entity with row-level access only the records the caller's access list allows are used. A field that requires a read Role is null (or left out, when not selected) unless the caller holds that Role, and the caller cannot filter or sort on it. A joined value is null when there is no matching record, when the caller may not read it, or when it lacks the field: these cases look the same, so a join never reveals that a hidden record exists.
+|
+|**Checking a body**
+|
+|Creating, updating or validating a Dynamic Query checks it against the Dynamic Entity definitions of its space, and `POST /obp/v7.0.0/management/dynamic-resource-docs/compile` does the same with `programming_lang` `Query`. Problems are reported as ${ErrorMessages.DynamicQueryInvalid.takeWhile(_ != ':')}.
+|
+|`POST /obp/v7.0.0/management/dynamic-resource-docs/explain` shows how a Dynamic Query would be answered, without reading any record, so its author can check that the SQL is sane and that access is what they expect: each read it would make, in order, with the SQL OBP builds for it (every value shown as `?`) or a description when it goes through the record provider; every Dynamic Entity it reads, with its read Role and whether the caller may read it; every read-restricted field it touches; and the refusal a caller would get. It can explain the query for the requesting User or for a caller who is not logged in, and with the parameters a caller would add. The API Manager's Explain button uses it.
+|
""".stripMargin)
glossaryItems += GlossaryItem(
@@ -4234,7 +4315,7 @@ object Glossary extends MdcLoggable {
|
|**Why**
|
-|A Dynamic Resource Doc method body, a Connector Method or a Dynamic Message Doc is user-supplied code compiled and run inside the OBP-API JVM, with the connector credentials and reach of the whole instance. The sandbox is not a meaningful second line of defence, so the primary control is that the person who writes the code (the *maker*) can never make it live alone: a different User holding the Role `CanApproveDynamicChangeRequest` (the *checker*) reviews the exact definition and approves it.
+|A Dynamic Resource Doc method body, a Connector Method or a Dynamic Message Doc is user-supplied code compiled and run inside the OBP-API JVM, with the connector credentials and reach of the whole instance. The primary control is that the person who writes the code (the *maker*) can never make it live alone: a different User holding the Role `CanApproveDynamicChangeRequest` (the *checker*) reviews the exact definition and approves it.
|
|**How it works when approval is on**
|
@@ -4242,7 +4323,7 @@ object Glossary extends MdcLoggable {
|
|2) The checker reads the request (`GET /obp/v7.0.0/management/dynamic-change-requests/CHANGE_REQUEST_ID`, which returns the proposed and the current payload side by side) and approves it by quoting its `payload_hash`, the SHA-256 of the exact body, on `POST .../approval`. Only then is the change applied. OBP refuses an approval from the User who made the request (`OBP-30279`).
|
-|3) Content is approved, not records. Any later edit produces a new hash and needs a new approval. The runtime compiles and serves only rows whose code hash equals the hash a checker approved. The code hash covers the programming language as well as the method body, and is recomputed from the row each time, so a row whose body or language is edited directly in the database does not run.
+|3) An approval covers the exact code. Any later edit produces a new hash and needs a new approval. The runtime compiles and serves only rows whose code hash equals the hash a checker approved. The code hash covers the programming language as well as the method body, and is recomputed from the row each time, so a row whose body or language is edited directly in the database does not run.
|
|4) Deactivating an artefact is a direct action by a single checker (`POST .../deactivation`), with no request: four eyes to enable, one pair to disable. Enabling it again goes through a request.
|
diff --git a/obp-api/src/main/scala/code/api/util/NewStyle.scala b/obp-api/src/main/scala/code/api/util/NewStyle.scala
index 57b3e419cf..21442b30d5 100644
--- a/obp-api/src/main/scala/code/api/util/NewStyle.scala
+++ b/obp-api/src/main/scala/code/api/util/NewStyle.scala
@@ -3528,14 +3528,15 @@ object NewStyle extends MdcLoggable{
/**
* Invalidate the Redis-backed resource-doc caches whose contents include
- * dynamic-entity documentation (the `dynamic` and `all` views). Bumping the
+ * dynamic documentation (the `dynamic` and `all` views). Bumping the
* namespace version orphans every cached key under that namespace, so the
* next `/resource-docs` request regenerates from the database instead of
* serving a pre-change snapshot (TTL is 1 hour by default).
*
- * Call after a dynamic entity is created, updated, or deleted. The static
- * resource-doc / swagger caches are not touched because dynamic entities
- * never appear in them.
+ * Call after a Dynamic Entity, Dynamic Endpoint or Dynamic Resource Doc is
+ * created, updated, or deleted: each one generates resource docs. The static
+ * resource-doc / swagger caches are not touched because dynamic docs only
+ * reach them through the content=all documents, which are left to their TTL.
*/
private def invalidateDynamicResourceDocCaches(): Unit = {
Constant.incrementCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE)
@@ -3617,7 +3618,13 @@ object NewStyle extends MdcLoggable{
}
} yield {
if (deleteEntitleMentResult) {
- DynamicEntityInfo.roleNames(entity.entityName, entity.bankId).foreach(ApiRole.removeDynamicApiRole(_))
+ // The Role names carry no bank, so while an entity of this name is left in another space
+ // they are still its Roles and stay registered.
+ val nameStillUsed = DynamicEntityProvider.connectorMethodProvider.vend
+ .getDynamicEntities(None, returnBothBankAndSystemLevel = true)
+ .exists(_.entityName == entity.entityName)
+ if (!nameStillUsed)
+ DynamicEntityInfo.roleNames(entity.entityName, entity.bankId).foreach(ApiRole.removeDynamicApiRole(_))
// Cascade row-level ACL rows for this entity (§7) — safety net for any rows not already
// cleaned up by per-row delete; no-op for non-row-level entities.
code.DynamicData.DynamicDataAccessProvider.provider.vend.deleteAllForEntity(entity.bankId, entity.entityName)
@@ -3894,7 +3901,9 @@ object NewStyle extends MdcLoggable{
def createDynamicEndpoint(bankId:Option[String], userId: String, swaggerString: String, callContext: Option[CallContext]): OBPReturnType[DynamicEndpointT] = {
validateBankId(bankId, callContext)
Future {
- (DynamicEndpointProvider.connectorMethodProvider.vend.create(bankId: Option[String], userId, swaggerString), callContext)
+ val created = DynamicEndpointProvider.connectorMethodProvider.vend.create(bankId: Option[String], userId, swaggerString)
+ if (created.isDefined) invalidateDynamicResourceDocCaches()
+ (created, callContext)
} map {
i => (connectorEmptyResponse(i._1, callContext), i._2)
}
@@ -3903,7 +3912,9 @@ object NewStyle extends MdcLoggable{
def updateDynamicEndpointHost(bankId: Option[String], userId: String, swaggerString: String, callContext: Option[CallContext]): OBPReturnType[DynamicEndpointT] = {
validateBankId(bankId, callContext)
Future {
- (DynamicEndpointProvider.connectorMethodProvider.vend.updateHost(bankId, userId, swaggerString), callContext)
+ val updated = DynamicEndpointProvider.connectorMethodProvider.vend.updateHost(bankId, userId, swaggerString)
+ if (updated.isDefined) invalidateDynamicResourceDocCaches()
+ (updated, callContext)
} map {
i => (connectorEmptyResponse(i._1, callContext), i._2)
}
@@ -3958,6 +3969,7 @@ object NewStyle extends MdcLoggable{
Full(false)
}
} yield {
+ if (deleteSuccess == Full(true)) invalidateDynamicResourceDocCaches()
deleteSuccess
}
}
@@ -4587,6 +4599,7 @@ object NewStyle extends MdcLoggable{
// provenance is taken from the authenticated CallContext user, never from the request body
val createdByUserId = callContext.flatMap(_.user).map(_.userId)
val newInternalConnector = DynamicResourceDocProvider.provider.vend.create(bankId, dynamicResourceDoc, createdByUserId)
+ if (newInternalConnector.isDefined) invalidateDynamicResourceDocCaches()
val errorMsg = s"$UnknownError Can not create Dynamic Resource Doc in the backend. "
(unboxFullOrFail(newInternalConnector, callContext, errorMsg, 400), callContext)
}
@@ -4595,6 +4608,7 @@ object NewStyle extends MdcLoggable{
Future {
val updatedByUserId = callContext.flatMap(_.user).map(_.userId)
val updatedConnectorMethod = DynamicResourceDocProvider.provider.vend.update(bankId, entity, updatedByUserId)
+ if (updatedConnectorMethod.isDefined) invalidateDynamicResourceDocCaches()
val errorMsg = s"$UnknownError Can not update Dynamic Resource Doc in the backend. "
(unboxFullOrFail(updatedConnectorMethod, callContext, errorMsg, 400), callContext)
}
@@ -4620,6 +4634,7 @@ object NewStyle extends MdcLoggable{
def deleteJsonDynamicResourceDocById(bankId: Option[String], dynamicResourceDocId: String, callContext: Option[CallContext]): OBPReturnType[Boolean] =
Future {
val dynamicResourceDoc = DynamicResourceDocProvider.provider.vend.deleteById(bankId, dynamicResourceDocId)
+ if (dynamicResourceDoc == Full(true)) invalidateDynamicResourceDocCaches()
(unboxFullOrFail(dynamicResourceDoc, callContext, s"$DynamicResourceDocDeleteError Current DYNAMIC_RESOURCE_DOC_ID(${dynamicResourceDocId})", 400), callContext)
}
diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sResourceDocs.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sResourceDocs.scala
index da7f1c94a1..a7a0c86f67 100644
--- a/obp-api/src/main/scala/code/api/util/http4s/Http4sResourceDocs.scala
+++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sResourceDocs.scala
@@ -29,7 +29,7 @@ package code.api.util.http4s
import org.json4s._
import cats.effect.IO
-import code.api.Constant.HostName
+import code.api.Constant.{DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID, HostName}
import code.api.ResourceDocs1_4_0.{ResourceDocs140, ResourceDocs300, ResourceDocsAPIMethodsUtil}
import code.api.ResponseHeader
import code.api.cache.Caching
@@ -73,6 +73,8 @@ import code.api.util.ApiTag.ResourceDocTag
* GET /obp/*/resource-docs/{API_VERSION}/openapi
* GET /obp/*/resource-docs/{API_VERSION}/openapi.yaml
* GET /obp/*/banks/{BANK_ID}/resource-docs/{API_VERSION}/obp
+ * GET /obp/*/banks/{BANK_ID}/resource-docs/{API_VERSION}/openapi
+ * GET /obp/*/banks/{BANK_ID}/resource-docs/{API_VERSION}/openapi.yaml
* GET /obp/*/message-docs/{CONNECTOR}/swagger2.0
*
* Wired into `Http4sApp.baseServices` BEFORE the Lift bridge, so requests are
@@ -230,6 +232,32 @@ object Http4sResourceDocs extends MdcLoggable {
}
}
+ // ─── Cache for rendered swagger / OpenAPI documents ──────────────────────
+
+ /**
+ * This function picks the cache a rendered swagger or OpenAPI document is kept in, returned as a
+ * getter and a setter.
+ *
+ * A document of dynamic docs only (content=dynamic) goes in the dynamic resource docs cache. That
+ * cache's namespace is bumped whenever a Dynamic Entity is created, updated or deleted
+ * (NewStyle.function.invalidateDynamicResourceDocCaches), so the next request rebuilds the
+ * document. The static swagger cache is never bumped that way, and a dynamic document kept there
+ * went on omitting a new Dynamic Entity for the rest of its TTL. Every other document stays in the
+ * static swagger cache.
+ *
+ * The key is prefixed with the format in the dynamic cache, because the obp format keeps its own
+ * content=dynamic document there under a key built from the same arguments.
+ */
+ private def renderedDocCache(
+ format: String,
+ contentParam: Option[ContentParam]
+ ): (String => Option[String], (String, String) => Unit) =
+ if (contentParam.contains(DYNAMIC))
+ (key => Caching.getDynamicResourceDocCache(s"$format:$key"),
+ (key, value) => Caching.setDynamicResourceDocCache(s"$format:$key", value))
+ else
+ (Caching.getStaticSwaggerDocCache, Caching.setStaticSwaggerDocCache)
+
// ─── Common parameter validation ─────────────────────────────────────────
// Mirrors the parameter-validation branches in the Lift handlers.
@@ -407,7 +435,8 @@ object Http4sResourceDocs extends MdcLoggable {
params.apiCollectionId,
Some(isVersion4OrHigher)
)
- val cached = Caching.getStaticSwaggerDocCache(cacheKey)
+ val (renderedDocCacheGet, renderedDocCacheSet) = renderedDocCache("swagger", params.contentParam)
+ val cached = renderedDocCacheGet(cacheKey)
val jv: JValue =
if (cached.isDefined) json.parse(cached.get)
else {
@@ -427,7 +456,7 @@ object Http4sResourceDocs extends MdcLoggable {
impl.getAllResourceDocsObpCached(requestedApiVersionString, filters.tags, filters.functions, params.locale, params.contentParam, isVersion4OrHigher).head.resource_docs
}
}
- impl.convertResourceDocsToSwaggerJvalueAndSetCache(cacheKey, requestedApiVersionString, resourceDocsJsonFiltered)
+ impl.convertResourceDocsToSwaggerJvalueAndSetCache(cacheKey, requestedApiVersionString, resourceDocsJsonFiltered, renderedDocCacheSet)
}
Right(jv)
} catch {
@@ -480,7 +509,8 @@ object Http4sResourceDocs extends MdcLoggable {
params.apiCollectionId,
Some(isVersion4OrHigher)
)
- val cached = Caching.getStaticSwaggerDocCache(cacheKey)
+ val (renderedDocCacheGet, renderedDocCacheSet) = renderedDocCache("openapi31", params.contentParam)
+ val cached = renderedDocCacheGet(cacheKey)
val jv: JValue =
if (cached.isDefined) json.parse(cached.get)
else {
@@ -500,7 +530,7 @@ object Http4sResourceDocs extends MdcLoggable {
impl.getAllResourceDocsObpCached(requestedApiVersionString, filters.tags, filters.functions, params.locale, params.contentParam, isVersion4OrHigher).head.resource_docs
}
}
- impl.convertResourceDocsToOpenAPI31JvalueAndSetCache(cacheKey, requestedApiVersionString, resourceDocsJsonFiltered)
+ impl.convertResourceDocsToOpenAPI31JvalueAndSetCache(cacheKey, requestedApiVersionString, resourceDocsJsonFiltered, renderedDocCacheSet)
}
Right(jv)
} catch {
@@ -550,7 +580,8 @@ object Http4sResourceDocs extends MdcLoggable {
params.apiCollectionId,
Some(isVersion4OrHigher)
)
- val cached = Caching.getStaticSwaggerDocCache(cacheKey)
+ val (renderedDocCacheGet, renderedDocCacheSet) = renderedDocCache("openapi31yaml", params.contentParam)
+ val cached = renderedDocCacheGet(cacheKey)
val yamlString: String =
if (cached.isDefined) cached.get
else {
@@ -570,7 +601,7 @@ object Http4sResourceDocs extends MdcLoggable {
impl.getAllResourceDocsObpCached(requestedApiVersionString, filters.tags, filters.functions, params.locale, params.contentParam, isVersion4OrHigher).head.resource_docs
}
}
- impl.convertResourceDocsToOpenAPI31YAMLAndSetCache(cacheKey, requestedApiVersionString, resourceDocsJsonFiltered)
+ impl.convertResourceDocsToOpenAPI31YAMLAndSetCache(cacheKey, requestedApiVersionString, resourceDocsJsonFiltered, renderedDocCacheSet)
}
Right(yamlString)
} catch {
@@ -580,6 +611,56 @@ object Http4sResourceDocs extends MdcLoggable {
}
}
+ // ─── Bank level handlers: GET /obp/*/banks/{BANK_ID}/resource-docs/{API_VERSION}/... ─
+ //
+ // These routes document the dynamic things that belong to one space: the Dynamic Entities,
+ // Dynamic Endpoints and Dynamic Resource Docs of one bank, or of the system space when BANK_ID
+ // is SYS (Constant.DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID). Static endpoints belong to no bank, so
+ // a bank level document only ever holds dynamic docs, and the `content` parameter does not
+ // apply. The three formats (obp, openapi, openapi.yaml) share one gate, bankLevelGate.
+
+ /**
+ * This function runs the checks every bank level resource-docs route makes before building its
+ * document: the optional role check (only when resource_docs_requires_role=true), then the
+ * check that the space exists, then the route's own parameter checks (`validate`). An error
+ * from the last two is rendered with `errorResponse`, so the YAML route can answer in plain text.
+ */
+ private def bankLevelGate(
+ req: Request[IO],
+ prefix: String,
+ bankIdStr: String,
+ errorResponse: (Status, String) => IO[Response[IO]]
+ )(validate: => Option[(Status, String)])(body: => IO[Response[IO]]): IO[Response[IO]] =
+ withOptionalRoleCheck(req, prefix, bankIdStr, canReadDynamicResourceDocsAtOneBank :: Nil,
+ UserHasMissingRoles + canReadDynamicResourceDocsAtOneBank.toString) {
+ if (!spaceExists(bankIdStr)) errorResponse(Status.NotFound, s"$BankNotFound Current BANK_ID = $bankIdStr")
+ else validate match {
+ case Some((status, message)) => errorResponse(status, message)
+ case None => body
+ }
+ }
+
+ /**
+ * This function says whether a space exists. SYS is the system space rather than a bank row, so
+ * it exists without a bank lookup; any other value must be the id of a bank.
+ */
+ private def spaceExists(bankIdStr: String): Boolean =
+ bankIdStr == DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID ||
+ code.bankconnectors.Connector.connector.vend.getBankLegacy(BankId(bankIdStr), None).map(_._1).isDefined
+
+ /** The dynamic docs of one space, filtered by the request's tags and functions. */
+ private def bankLevelDynamicDocs(
+ params: ParsedParams,
+ prefix: String,
+ bankIdStr: String,
+ isVersion4OrHigher: Boolean
+ ): List[JSONFactory1_4_0.ResourceDocJson] = {
+ val filters = ResourceDocFilters.forResourceDocs(params.tagValues, params.functionValues)
+ implForPrefix(prefix)
+ .getResourceDocsObpDynamicCached(filters.tags, filters.functions, params.locale, Some(bankIdStr), isVersion4OrHigher)
+ .head.resource_docs
+ }
+
// ─── Handler: GET /obp/*/banks/{BANK_ID}/resource-docs/{API_VERSION}/obp ─
private def handleGetBankLevelDynamicResourceDocsObp(
@@ -589,30 +670,20 @@ object Http4sResourceDocs extends MdcLoggable {
requestedApiVersionString: String
): IO[Response[IO]] = {
val params = parseParams(req)
- withOptionalRoleCheck(req, prefix, bankIdStr, canReadDynamicResourceDocsAtOneBank :: Nil,
- UserHasMissingRoles + canReadDynamicResourceDocsAtOneBank.toString) {
- // Bank-level handler ALWAYS requires the bank to exist. Use the legacy connector lookup
- // synchronously; it does its own 404 if absent.
- val bankBox: Box[com.openbankproject.commons.model.Bank] =
- code.bankconnectors.Connector.connector.vend.getBankLegacy(BankId(bankIdStr), None).map(_._1)
- if (bankBox.isEmpty) errorJson(Status.NotFound, s"$BankNotFound Current BANK_ID = $bankIdStr")
- else {
- val localeError: Option[String] = params.locale match {
- case Some(l) if APIUtil.obpLocaleValidation(l) != SILENCE_IS_GOLDEN =>
- Some(s"$InvalidLocale Current Locale is $l")
- case _ => None
- }
- val versionError: Option[String] =
- try { ApiVersionUtils.valueOf(requestedApiVersionString); None }
- catch { case _: Throwable => Some(s"$InvalidApiVersionString $requestedApiVersionString") }
- localeError.orElse(versionError) match {
- case Some(msg) => errorJson(Status.BadRequest, msg)
- case None =>
- IO(buildBankLevelResourceDocsJson(params, prefix, bankIdStr, requestedApiVersionString)).flatMap {
- case Right(body) => jsonResponse(Status.Ok, body)
- case Left((s, m)) => errorJson(s, m)
- }
- }
+ bankLevelGate(req, prefix, bankIdStr, errorJson) {
+ val localeError: Option[String] = params.locale match {
+ case Some(l) if APIUtil.obpLocaleValidation(l) != SILENCE_IS_GOLDEN =>
+ Some(s"$InvalidLocale Current Locale is $l")
+ case _ => None
+ }
+ val versionError: Option[String] =
+ try { ApiVersionUtils.valueOf(requestedApiVersionString); None }
+ catch { case _: Throwable => Some(s"$InvalidApiVersionString $requestedApiVersionString") }
+ localeError.orElse(versionError).map(Status.BadRequest -> _)
+ } {
+ IO(buildBankLevelResourceDocsJson(params, prefix, bankIdStr, requestedApiVersionString)).flatMap {
+ case Right(body) => jsonResponse(Status.Ok, body)
+ case Left((s, m)) => errorJson(s, m)
}
}
}
@@ -639,7 +710,7 @@ object Http4sResourceDocs extends MdcLoggable {
val jv: JValue =
if (cached.isDefined) json.parse(cached.get)
else {
- val rdJson = impl.getResourceDocsObpDynamicCached(filters.tags, filters.functions, params.locale, None, isVersion4OrHigher = false).head
+ val rdJson = impl.getResourceDocsObpDynamicCached(filters.tags, filters.functions, params.locale, Some(bankIdStr), isVersion4OrHigher = false).head
val response = resourceDocsJsonToJsonResponse(rdJson)
Caching.setDynamicResourceDocCache(cacheKey, json.compactRender(response))
response
@@ -652,6 +723,88 @@ object Http4sResourceDocs extends MdcLoggable {
}
}
+ // ─── Handler: GET /obp/*/banks/{BANK_ID}/resource-docs/{API_VERSION}/openapi ─
+
+ private def handleGetBankLevelDynamicResourceDocsOpenAPI31(
+ req: Request[IO],
+ prefix: String,
+ bankIdStr: String,
+ requestedApiVersionString: String
+ ): IO[Response[IO]] = {
+ val params = parseParams(req)
+ bankLevelGate(req, prefix, bankIdStr, errorJson) {
+ validateBasicParams(params).orElse(validateVersionAndLocale(requestedApiVersionString, params.locale).left.toOption)
+ } {
+ IO(buildBankLevelOpenApi31(params, prefix, bankIdStr, requestedApiVersionString, yaml = false)).flatMap {
+ case Right(body) => IO.pure(Response[IO](Status.Ok).withEntity(body).withContentType(jsonContentType))
+ case Left((s, m)) => errorJson(s, m)
+ }
+ }
+ }
+
+ // ─── Handler: GET /obp/*/banks/{BANK_ID}/resource-docs/{API_VERSION}/openapi.yaml ─
+
+ private def handleGetBankLevelDynamicResourceDocsOpenAPI31Yaml(
+ req: Request[IO],
+ prefix: String,
+ bankIdStr: String,
+ requestedApiVersionString: String
+ ): IO[Response[IO]] = {
+ val params = parseParams(req)
+ bankLevelGate(req, prefix, bankIdStr, plainTextResponse) {
+ validateBasicParams(params).orElse(validateVersionAndLocale(requestedApiVersionString, params.locale).left.toOption)
+ } {
+ IO(buildBankLevelOpenApi31(params, prefix, bankIdStr, requestedApiVersionString, yaml = true)).flatMap {
+ case Right(yamlString) => yamlResponse(yamlString)
+ case Left((s, m)) => plainTextResponse(s, m)
+ }
+ }
+ }
+
+ /**
+ * This function builds the OpenAPI 3.1 document of one space, as compact JSON or as YAML. The
+ * cache key carries the space and the format, so neither can be served the other's document.
+ * The document is cached with the dynamic resource docs TTL, not the static swagger one, because
+ * it changes whenever a Dynamic Entity, Dynamic Endpoint or Dynamic Resource Doc is added.
+ */
+ private def buildBankLevelOpenApi31(
+ params: ParsedParams,
+ prefix: String,
+ bankIdStr: String,
+ requestedApiVersionString: String,
+ yaml: Boolean
+ ): Either[(Status, String), String] = {
+ try {
+ val filters = ResourceDocFilters.forResourceDocs(params.tagValues, params.functionValues)
+ val format = if (yaml) "openapi31yaml" else "openapi31"
+ val cacheKey = APIUtil.createResourceDocCacheKey(
+ Some(s"$format-bank:$bankIdStr"),
+ requestedApiVersionString,
+ filters,
+ params.locale,
+ None,
+ None,
+ Some(true)
+ )
+ val cached = Caching.getDynamicResourceDocCache(cacheKey)
+ if (cached.isDefined) Right(cached.get)
+ else {
+ val docs = bankLevelDynamicDocs(params, prefix, bankIdStr, isVersion4OrHigher = true)
+ val openApiDoc = code.api.ResourceDocs1_4_0.OpenAPI31JSONFactory.createOpenAPI31Json(docs, requestedApiVersionString, HostName)
+ val openApiJValue = code.api.ResourceDocs1_4_0.OpenAPI31JSONFactory.OpenAPI31JsonFormats.toJValue(openApiDoc)
+ val rendered =
+ if (yaml) YAMLUtils.jValueToYAMLSafe(openApiJValue, "# Error converting to YAML")
+ else json.compactRender(openApiJValue)
+ Caching.setDynamicResourceDocCache(cacheKey, rendered)
+ Right(rendered)
+ }
+ } catch {
+ case e: Throwable =>
+ logger.error(s"Http4sResourceDocs.buildBankLevelOpenApi31 failed: ${e.getMessage}", e)
+ Left(Status.BadRequest -> s"$UnknownError Can not create the OpenAPI document for BANK_ID $bankIdStr.")
+ }
+ }
+
// ─── Handler: GET /obp/*/message-docs/{CONNECTOR}/swagger2.0 ─────────────
private def handleGetMessageDocsSwagger(
@@ -758,6 +911,14 @@ object Http4sResourceDocs extends MdcLoggable {
case req @ GET -> Root / "obp" / prefix / "banks" / bankIdStr / "resource-docs" / requestedApiVersionString / "obp" =>
timed(req, "getBankLevelDynamicResourceDocsObp")(handleGetBankLevelDynamicResourceDocsObp(req, prefix, bankIdStr, requestedApiVersionString))
+ case req @ GET -> Root / "obp" / prefix / "banks" / bankIdStr / "resource-docs" / requestedApiVersionString / "openapi" =>
+ timed(req, "getBankLevelDynamicResourceDocsOpenAPI31")(handleGetBankLevelDynamicResourceDocsOpenAPI31(req, prefix, bankIdStr, requestedApiVersionString))
+
+ // Like the instance wide YAML route, the YAML form has no ResourceDoc of its own (the openapi
+ // one describes both), so it has no operation id and is not timed.
+ case req @ GET -> Root / "obp" / prefix / "banks" / bankIdStr / "resource-docs" / requestedApiVersionString / "openapi.yaml" =>
+ handleGetBankLevelDynamicResourceDocsOpenAPI31Yaml(req, prefix, bankIdStr, requestedApiVersionString)
+
case req @ GET -> Root / "obp" / _ / "message-docs" / connector / "swagger2.0" =>
timedAs(req, APIUtil.buildOperationId(ApiVersion.v3_1_0, "getMessageDocsSwagger"), ApiVersion.v3_1_0.apiShortVersion)(
handleGetMessageDocsSwagger(req, connector))
diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala
index 7a3502712d..09c138b3af 100644
--- a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala
+++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala
@@ -229,6 +229,21 @@ object Http4sRequestAttributes {
WriteMetricUtil.writeEndpointMetric(responseBody, Some(ccLight))
}
+ /**
+ * This records the endpoint metric for a response that was built outside these helpers, such as
+ * the handler of a Dynamic Resource Doc, which builds its own Response. Every other endpoint gets
+ * its metric from the helper it runs in; without this, calls to those handlers left no metric row.
+ *
+ * The body is read only when metrics are written at all. Those handlers build their bodies from
+ * in-memory strings, so reading it here does not consume what the client receives.
+ */
+ def recordMetricFor(response: Response[IO])(implicit cc: CallContext): IO[Response[IO]] =
+ if (!code.metrics.MetricsProps.writeMetrics) IO.pure(response)
+ else response.bodyText.compile.string.flatMap { text =>
+ val body: Any = scala.util.Try(com.openbankproject.commons.util.JsonAliases.parse(text)).getOrElse(text)
+ recordMetric(body, response)
+ }.as(response)
+
/**
* Execute Future-based business logic and return JSON response.
* Returns 200 OK on success, converts errors via ErrorResponseConverter.
diff --git a/obp-api/src/main/scala/code/api/util/http4s/SelfServiceRateLimitMiddleware.scala b/obp-api/src/main/scala/code/api/util/http4s/SelfServiceRateLimitMiddleware.scala
index 77694cccb0..1e319919bf 100644
--- a/obp-api/src/main/scala/code/api/util/http4s/SelfServiceRateLimitMiddleware.scala
+++ b/obp-api/src/main/scala/code/api/util/http4s/SelfServiceRateLimitMiddleware.scala
@@ -94,7 +94,7 @@ object SelfServiceRateLimitMiddleware extends MdcLoggable {
// monitoring polls it. Shadow mode unless the scope's own mode prop is set
// (SelfServiceRateLimiter.shadowUnlessSetScopes).
Entry("documentation", Method.GET, "^/obp/[^/]+/resource-docs/[^/]+/(obp|swagger|openapi|openapi\\.yaml)$".r),
- Entry("documentation", Method.GET, "^/obp/[^/]+/banks/[^/]+/resource-docs/[^/]+/obp$".r),
+ Entry("documentation", Method.GET, "^/obp/[^/]+/banks/[^/]+/resource-docs/[^/]+/(obp|openapi|openapi\\.yaml)$".r),
Entry("documentation", Method.GET, "^/obp/[^/]+/message-docs/[^/]+(/json-schema|/swagger2\\.0)?$".r),
Entry("documentation", Method.GET, "^/obp/[^/]+/api/(glossary(/[^/]+)?|tags|versions|error-messages|popular-endpoints)$".r),
Entry("documentation", Method.GET, "^/obp/[^/]+/endpoints/(json-schema-validations|authentication-type-validations)$".r)
diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala
index 961ef238cf..0d7c41e55a 100644
--- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala
+++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala
@@ -1826,8 +1826,9 @@ object Http4s400 {
"Update System Level Dynamic Entity",
s"""Update a system level DynamicEntity.
|
- |If the entity already has data, only schema-compatible changes are accepted: the entity name, the set of
- |property names and each property's `type` must stay the same, and `required` may not grow. Changing
+ |If the entity already has data, only schema-compatible changes are accepted: the entity name must stay the
+ |same, every existing property must keep its name and `type`, and `required` may not grow. New optional
+ |properties may be added. Changing
|`indexed`, `index`, `example`, `description`, `minLength`, `maxLength` and the read/write role settings is
|allowed — this is how indexing is switched on for an existing entity (see DE_indexing). A structural change
|returns `$DynamicEntityUpdateNotSchemaCompatible` until the data is deleted.
@@ -1862,8 +1863,9 @@ object Http4s400 {
"Update Bank Level Dynamic Entity",
s"""Update a Bank Level DynamicEntity.
|
- |If the entity already has data, only schema-compatible changes are accepted: the entity name, the set of
- |property names and each property's `type` must stay the same, and `required` may not grow. Changing
+ |If the entity already has data, only schema-compatible changes are accepted: the entity name must stay the
+ |same, every existing property must keep its name and `type`, and `required` may not grow. New optional
+ |properties may be added. Changing
|`indexed`, `index`, `example`, `description`, `minLength`, `maxLength` and the read/write role settings is
|allowed — this is how indexing is switched on for an existing entity (see DE_indexing). A structural change
|returns `$DynamicEntityUpdateNotSchemaCompatible` until the data is deleted.
@@ -1989,8 +1991,9 @@ object Http4s400 {
"Update My Dynamic Entity",
s"""Update my DynamicEntity specified by DYNAMIC_ENTITY_ID.
|
- |If the entity already has data, only schema-compatible changes are accepted: the entity name, the set of
- |property names and each property's `type` must stay the same, and `required` may not grow. Changing
+ |If the entity already has data, only schema-compatible changes are accepted: the entity name must stay the
+ |same, every existing property must keep its name and `type`, and `required` may not grow. New optional
+ |properties may be added. Changing
|`indexed`, `index`, `example`, `description`, `minLength`, `maxLength` and the read/write role settings is
|allowed — this is how indexing is switched on for an existing entity (see DE_indexing). A structural change
|returns `$DynamicEntityUpdateNotSchemaCompatible` until the data is deleted.
@@ -9586,6 +9589,9 @@ object Http4s400 {
cc = Some(cc)) {
code.api.dynamic.endpoint.helper.CompiledObjects.isSupportedLanguage(body.programmingLang)
}
+ _ <- code.util.Helper.booleanToFuture(s"$DynamicQueryInvalid${code.api.dynamic.endpoint.helper.CompiledObjects.queryVerbMessage}", cc = Some(cc)) {
+ code.api.dynamic.endpoint.helper.CompiledObjects.verbAllowed(body.programmingLang, body.requestVerb)
+ }
} yield ()
}
@@ -9604,9 +9610,9 @@ object Http4s400 {
code.util.Helper.booleanToFuture(s"$InvalidJsonFormat ${tooLong.mkString("; ")}", cc = Some(cc)) { tooLong.isEmpty }.map(_ => ())
}
- private def compileDynamicResourceDoc(body: JsonDynamicResourceDoc, cc: CallContext): Unit = {
+ private def compileDynamicResourceDoc(bankId: Option[String], body: JsonDynamicResourceDoc, cc: CallContext): Unit = {
try {
- CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency()
+ CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang, bankId).validateDependency()
} catch {
case e: JsonResponseException => throw e
case e: Exception =>
@@ -9633,14 +9639,17 @@ object Http4s400 {
private def createDynamicResourceDocImpl(bankId: Option[String], rawBody: String, cc: CallContext): Future[(Any, Int)] = {
for {
- _ <- code.util.Helper.booleanToFuture(DynamicCodeExecutionDisabled, cc = Some(cc)) { DynamicUtil.dynamicCodeExecutionEnabled }
body <- NewStyle.function.tryons(
s"$InvalidJsonFormat The Json body should be the ${classOf[JsonDynamicResourceDoc].getSimpleName}",
400, Some(cc)) {
com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[JsonDynamicResourceDoc]
}
+ // A Dynamic Query runs no user code, so only the other languages need dynamic code to be enabled.
+ _ <- code.util.Helper.booleanToFuture(DynamicCodeExecutionDisabled, cc = Some(cc)) {
+ DynamicUtil.dynamicCodeExecutionEnabled || CompiledObjects.isQuery(body.programmingLang)
+ }
_ <- validateDynamicResourceDocBody(body, cc)
- _ = compileDynamicResourceDoc(body, cc)
+ _ = compileDynamicResourceDoc(bankId.orElse(body.bankId), body, cc)
(isExists, callContext) <- NewStyle.function.isJsonDynamicResourceDocExists(
bankId, body.requestVerb, body.requestUrl, Some(cc))
_ <- code.util.Helper.booleanToFuture(
@@ -9654,14 +9663,17 @@ object Http4s400 {
private def updateDynamicResourceDocImpl(bankId: Option[String], dynamicResourceDocId: String, rawBody: String, cc: CallContext): Future[(Any, Int)] = {
for {
- _ <- code.util.Helper.booleanToFuture(DynamicCodeExecutionDisabled, cc = Some(cc)) { DynamicUtil.dynamicCodeExecutionEnabled }
body <- NewStyle.function.tryons(
s"$InvalidJsonFormat The Json body should be the ${classOf[JsonDynamicResourceDoc].getSimpleName}",
400, Some(cc)) {
com.openbankproject.commons.util.JsonAliases.parse(rawBody).extract[JsonDynamicResourceDoc]
}
+ // A Dynamic Query runs no user code, so only the other languages need dynamic code to be enabled.
+ _ <- code.util.Helper.booleanToFuture(DynamicCodeExecutionDisabled, cc = Some(cc)) {
+ DynamicUtil.dynamicCodeExecutionEnabled || CompiledObjects.isQuery(body.programmingLang)
+ }
_ <- validateDynamicResourceDocBody(body, cc)
- _ = compileDynamicResourceDoc(body, cc)
+ _ = compileDynamicResourceDoc(bankId.orElse(body.bankId), body, cc)
(_, callContext) <- NewStyle.function.getJsonDynamicResourceDocById(bankId, dynamicResourceDocId, Some(cc))
result <- interceptOrApply(DYNAMIC_RESOURCE_DOC, ChangeOp.UPDATE, Some(dynamicResourceDocId), 200, cc) {
NewStyle.function.updateJsonDynamicResourceDoc(
diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala
index 9c12faed85..37d1603a76 100644
--- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala
+++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala
@@ -4674,25 +4674,42 @@ object Http4s600 {
cc = Some(cc)) {
code.api.dynamic.endpoint.helper.CompiledObjects.isSupportedLanguage(body.programmingLang)
}
- } yield try {
- code.api.dynamic.endpoint.helper.CompiledObjects(
- body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency()
- ValidateDynamicResourceDocSuccessJsonV600(
- valid = true,
- message = s"Dynamic Resource Doc method body is valid ${body.programmingLang} and uses allowed dependencies.")
- } catch {
- case e: code.api.JsonResponseException =>
- val errorText = e.jsonResponse match {
- case code.api.util.APIUtil.JsonResponseExtractor(msg, _) => msg
- case _ => ""
- }
- ValidateDynamicResourceDocFailureJsonV600(
- valid = false, error = errorText, message = DynamicResourceDocMethodDependency,
- details = ValidateDynamicResourceDocErrorDetailsJsonV600(error_type = "DependencyError"))
- case e: Exception =>
- ValidateDynamicResourceDocFailureJsonV600(
- valid = false, error = Option(e.getMessage).getOrElse(""), message = DynamicCodeCompileFail,
- details = ValidateDynamicResourceDocErrorDetailsJsonV600(error_type = "CompilationError"))
+ _ <- Helper.booleanToFuture(s"$DynamicQueryInvalid${code.api.dynamic.endpoint.helper.CompiledObjects.queryVerbMessage}", cc = Some(cc)) {
+ code.api.dynamic.endpoint.helper.CompiledObjects.verbAllowed(body.programmingLang, body.requestVerb)
+ }
+ } yield {
+ val isQuery = code.api.dynamic.endpoint.helper.CompiledObjects.isQuery(body.programmingLang)
+ try {
+ code.api.dynamic.endpoint.helper.CompiledObjects(
+ body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang, body.bankId).validateDependency()
+ ValidateDynamicResourceDocSuccessJsonV600(
+ valid = true,
+ message =
+ if (isQuery) "Dynamic Query declaration is valid."
+ else s"Dynamic Resource Doc method body is valid ${body.programmingLang} and uses allowed dependencies.")
+ } catch {
+ // A Dynamic Query is checked against the entity definitions, not a dependency allowlist.
+ case e: code.api.JsonResponseException if isQuery =>
+ val errorText = e.jsonResponse match {
+ case code.api.util.APIUtil.JsonResponseExtractor(msg, _) => msg
+ case _ => ""
+ }
+ ValidateDynamicResourceDocFailureJsonV600(
+ valid = false, error = errorText, message = DynamicQueryInvalid,
+ details = ValidateDynamicResourceDocErrorDetailsJsonV600(error_type = "QueryError"))
+ case e: code.api.JsonResponseException =>
+ val errorText = e.jsonResponse match {
+ case code.api.util.APIUtil.JsonResponseExtractor(msg, _) => msg
+ case _ => ""
+ }
+ ValidateDynamicResourceDocFailureJsonV600(
+ valid = false, error = errorText, message = DynamicResourceDocMethodDependency,
+ details = ValidateDynamicResourceDocErrorDetailsJsonV600(error_type = "DependencyError"))
+ case e: Exception =>
+ ValidateDynamicResourceDocFailureJsonV600(
+ valid = false, error = Option(e.getMessage).getOrElse(""), message = DynamicCodeCompileFail,
+ details = ValidateDynamicResourceDocErrorDetailsJsonV600(error_type = "CompilationError"))
+ }
}
}
}
@@ -7322,7 +7339,7 @@ object Http4s600 {
|* Each property MUST include an `example` field with a valid example value.
|* Each property can optionally include `description` (markdown text), and for string types: `minLength` and `maxLength`.
|* Each property can optionally be marked queryable with `"indexed": true` — only indexed fields may be used in the list endpoint's filter/sort query parameters (and a `reference:` field must be indexed to form a join edge). Add `"index": "spatial"` for a GeoJSON geometry index (only valid on a `json` field); the default when omitted is `"index": "scalar"` (B-tree).
- |* Each property can optionally declare **field-level access control**: `write_role_required`/`read_role_required` (booleans — auto-generate a per-field role) or `write_role`/`read_role` (name an explicit, shareable role). Write-restricted fields are not set via POST/PUT (their existing value is preserved) and are written only via the role-gated PATCH path; read-restricted fields are omitted from GET for callers lacking the read role.
+ |* Each property can optionally declare **field-level access control**: `write_role_required`/`read_role_required` (booleans — auto-generate a per-field role) or `write_role`/`read_role` (name an explicit, shareable role). Write-restricted fields are not set via POST/PUT (their existing value is preserved) and are written only via the role-gated PATCH path; read-restricted fields are omitted from GET for callers lacking the read role. `hide_field_from_public_access` (boolean) hides a field of a public entity from callers whose access comes only from its public access, while callers holding the entity's read role still see it.
|* Set `has_public_access` to `true` to generate read-only public endpoints (GET only, no authentication required) under `/public/`.
|* Set `auth_mode` to say who may hold the roles that guard the entity's data endpoints: `UserOnly` (default, the User's Entitlements), `ApplicationOnly` (the Consumer's Scopes), `UserOrApplication` (either) or `UserAndApplication` (both). Personal (`/my/`) endpoints always require a User. An entity with `has_personal_entity` cannot be `ApplicationOnly`.
|* Set `has_community_access` to `true` to generate read-only community endpoints (GET only, authentication required + CanGet role) under `/community/`. Community endpoints return ALL records (personal + non-personal from all users).
@@ -7395,7 +7412,7 @@ object Http4s600 {
|* Each property MUST include an `example` field with a valid example value.
|* Each property can optionally include `description` (markdown text), and for string types: `minLength` and `maxLength`.
|* Each property can optionally be marked queryable with `"indexed": true` — only indexed fields may be used in the list endpoint's filter/sort query parameters (and a `reference:` field must be indexed to form a join edge). Add `"index": "spatial"` for a GeoJSON geometry index (only valid on a `json` field); the default when omitted is `"index": "scalar"` (B-tree).
- |* Each property can optionally declare **field-level access control**: `write_role_required`/`read_role_required` (booleans — auto-generate a per-field role) or `write_role`/`read_role` (name an explicit, shareable role). Write-restricted fields are not set via POST/PUT (their existing value is preserved) and are written only via the role-gated PATCH path; read-restricted fields are omitted from GET for callers lacking the read role.
+ |* Each property can optionally declare **field-level access control**: `write_role_required`/`read_role_required` (booleans — auto-generate a per-field role) or `write_role`/`read_role` (name an explicit, shareable role). Write-restricted fields are not set via POST/PUT (their existing value is preserved) and are written only via the role-gated PATCH path; read-restricted fields are omitted from GET for callers lacking the read role. `hide_field_from_public_access` (boolean) hides a field of a public entity from callers whose access comes only from its public access, while callers holding the entity's read role still see it.
|* Set `has_public_access` to `true` to generate read-only public endpoints (GET only, no authentication required) under `/public/`.
|* Set `auth_mode` to say who may hold the roles that guard the entity's data endpoints: `UserOnly` (default, the User's Entitlements), `ApplicationOnly` (the Consumer's Scopes), `UserOrApplication` (either) or `UserAndApplication` (both). Personal (`/my/`) endpoints always require a User. An entity with `has_personal_entity` cannot be `ApplicationOnly`.
|* Set `has_community_access` to `true` to generate read-only community endpoints (GET only, authentication required + CanGet role) under `/community/`. Community endpoints return ALL records (personal + non-personal from all users).
@@ -7470,7 +7487,7 @@ object Http4s600 {
|* The `entity_name` must be lowercase with underscores (snake_case), e.g. `customer_preferences`. No uppercase letters or spaces allowed.
|* Each property can optionally include `description` (markdown text), and for string types: `minLength` and `maxLength`.
|* Each property can optionally be marked queryable with `"indexed": true` — only indexed fields may be used in the list endpoint's filter/sort query parameters (and a `reference:` field must be indexed to form a join edge). Add `"index": "spatial"` for a GeoJSON geometry index (only valid on a `json` field); the default when omitted is `"index": "scalar"` (B-tree).
- |* Each property can optionally declare **field-level access control**: `write_role_required`/`read_role_required` (booleans — auto-generate a per-field role) or `write_role`/`read_role` (name an explicit, shareable role). Write-restricted fields are not set via POST/PUT (their existing value is preserved) and are written only via the role-gated PATCH path; read-restricted fields are omitted from GET for callers lacking the read role.
+ |* Each property can optionally declare **field-level access control**: `write_role_required`/`read_role_required` (booleans — auto-generate a per-field role) or `write_role`/`read_role` (name an explicit, shareable role). Write-restricted fields are not set via POST/PUT (their existing value is preserved) and are written only via the role-gated PATCH path; read-restricted fields are omitted from GET for callers lacking the read role. `hide_field_from_public_access` (boolean) hides a field of a public entity from callers whose access comes only from its public access, while callers holding the entity's read role still see it.
|* Set `has_public_access` to `true` to generate read-only public endpoints (GET only, no authentication required) under `/public/`.
|* Set `auth_mode` to say who may hold the roles that guard the entity's data endpoints: `UserOnly` (default, the User's Entitlements), `ApplicationOnly` (the Consumer's Scopes), `UserOrApplication` (either) or `UserAndApplication` (both). Personal (`/my/`) endpoints always require a User. An entity with `has_personal_entity` cannot be `ApplicationOnly`.
|* Set `has_community_access` to `true` to generate read-only community endpoints (GET only, authentication required + CanGet role) under `/community/`. Community endpoints return ALL records (personal + non-personal from all users).
@@ -7534,7 +7551,7 @@ object Http4s600 {
|* The `entity_name` must be lowercase with underscores (snake_case), e.g. `customer_preferences`. No uppercase letters or spaces allowed.
|* Each property can optionally include `description` (markdown text), and for string types: `minLength` and `maxLength`.
|* Each property can optionally be marked queryable with `"indexed": true` — only indexed fields may be used in the list endpoint's filter/sort query parameters (and a `reference:` field must be indexed to form a join edge). Add `"index": "spatial"` for a GeoJSON geometry index (only valid on a `json` field); the default when omitted is `"index": "scalar"` (B-tree).
- |* Each property can optionally declare **field-level access control**: `write_role_required`/`read_role_required` (booleans — auto-generate a per-field role) or `write_role`/`read_role` (name an explicit, shareable role). Write-restricted fields are not set via POST/PUT (their existing value is preserved) and are written only via the role-gated PATCH path; read-restricted fields are omitted from GET for callers lacking the read role.
+ |* Each property can optionally declare **field-level access control**: `write_role_required`/`read_role_required` (booleans — auto-generate a per-field role) or `write_role`/`read_role` (name an explicit, shareable role). Write-restricted fields are not set via POST/PUT (their existing value is preserved) and are written only via the role-gated PATCH path; read-restricted fields are omitted from GET for callers lacking the read role. `hide_field_from_public_access` (boolean) hides a field of a public entity from callers whose access comes only from its public access, while callers holding the entity's read role still see it.
|* Set `has_public_access` to `true` to generate read-only public endpoints (GET only, no authentication required) under `/public/`.
|* Set `auth_mode` to say who may hold the roles that guard the entity's data endpoints: `UserOnly` (default, the User's Entitlements), `ApplicationOnly` (the Consumer's Scopes), `UserOrApplication` (either) or `UserAndApplication` (both). Personal (`/my/`) endpoints always require a User. An entity with `has_personal_entity` cannot be `ApplicationOnly`.
|* Set `has_community_access` to `true` to generate read-only community endpoints (GET only, authentication required + CanGet role) under `/community/`. Community endpoints return ALL records (personal + non-personal from all users).
@@ -7604,7 +7621,7 @@ object Http4s600 {
|* The `entity_name` must be lowercase with underscores (snake_case), e.g. `customer_preferences`. No uppercase letters or spaces allowed.
|* Each property can optionally include `description` (markdown text), and for string types: `minLength` and `maxLength`.
|* Each property can optionally be marked queryable with `"indexed": true` — only indexed fields may be used in the list endpoint's filter/sort query parameters (and a `reference:` field must be indexed to form a join edge). Add `"index": "spatial"` for a GeoJSON geometry index (only valid on a `json` field); the default when omitted is `"index": "scalar"` (B-tree).
- |* Each property can optionally declare **field-level access control**: `write_role_required`/`read_role_required` (booleans — auto-generate a per-field role) or `write_role`/`read_role` (name an explicit, shareable role). Write-restricted fields are not set via POST/PUT (their existing value is preserved) and are written only via the role-gated PATCH path; read-restricted fields are omitted from GET for callers lacking the read role.
+ |* Each property can optionally declare **field-level access control**: `write_role_required`/`read_role_required` (booleans — auto-generate a per-field role) or `write_role`/`read_role` (name an explicit, shareable role). Write-restricted fields are not set via POST/PUT (their existing value is preserved) and are written only via the role-gated PATCH path; read-restricted fields are omitted from GET for callers lacking the read role. `hide_field_from_public_access` (boolean) hides a field of a public entity from callers whose access comes only from its public access, while callers holding the entity's read role still see it.
|* Set `has_public_access` to `true` to generate read-only public endpoints (GET only, no authentication required) under `/public/`.
|* Set `auth_mode` to say who may hold the roles that guard the entity's data endpoints: `UserOnly` (default, the User's Entitlements), `ApplicationOnly` (the Consumer's Scopes), `UserOrApplication` (either) or `UserAndApplication` (both). Personal (`/my/`) endpoints always require a User. An entity with `has_personal_entity` cannot be `ApplicationOnly`.
|* Set `has_community_access` to `true` to generate read-only community endpoints (GET only, authentication required + CanGet role) under `/community/`. Community endpoints return ALL records (personal + non-personal from all users).
diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala
index 1d07bac1fc..53a552d01c 100644
--- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala
+++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala
@@ -6813,7 +6813,6 @@ object Http4s700 {
EndpointHelpers.withUser(req) { (u, cc) =>
import code.api.v7_0_0.JSONFactory700.{DynamicCompileErrorJsonV700, DynamicCompileResultJsonV700, DynamicResourceDocCompileJsonV700}
for {
- _ <- code.util.Helper.booleanToFuture(DynamicCodeExecutionDisabled, cc = Some(cc)) { code.api.util.DynamicUtil.dynamicCodeExecutionEnabled }
_ <- code.util.Helper.booleanToFuture(s"${code.api.util.ErrorMessages.TooManyRequests} at most $dynamicCompileCallsPerMinute dry-run compiles per minute per user", 429, Some(cc)) { allowDynamicCompile(u.userId) }
body <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the ${classOf[DynamicResourceDocCompileJsonV700].getSimpleName}", 400, Some(cc)) {
com.openbankproject.commons.util.JsonAliases.parse(cc.httpBody.getOrElse("")).extract[DynamicResourceDocCompileJsonV700]
@@ -6827,6 +6826,13 @@ object Http4s700 {
cc = Some(cc)) {
code.api.dynamic.endpoint.helper.CompiledObjects.isSupportedLanguage(programmingLang)
}
+ // A Dynamic Query runs no user code, so only the other languages need dynamic code to be enabled.
+ _ <- code.util.Helper.booleanToFuture(DynamicCodeExecutionDisabled, cc = Some(cc)) {
+ code.api.util.DynamicUtil.dynamicCodeExecutionEnabled || code.api.dynamic.endpoint.helper.CompiledObjects.isQuery(programmingLang)
+ }
+ _ <- code.util.Helper.booleanToFuture(s"${code.api.util.ErrorMessages.DynamicQueryInvalid}${code.api.dynamic.endpoint.helper.CompiledObjects.queryVerbMessage}", cc = Some(cc)) {
+ code.api.dynamic.endpoint.helper.CompiledObjects.verbAllowed(programmingLang, body.request_verb)
+ }
result <- Future {
val start = System.currentTimeMillis()
val problems = scala.util.Try(code.api.dynamic.endpoint.helper.CompiledObjects.compileProblems(body.example_request_body, body.success_response_body, body.method_body, programmingLang)) match {
@@ -6866,6 +6872,10 @@ object Http4s700 {
|A Java body is compiled as written, so `example_request_body` and `success_response_body` do not affect it. It must declare a public class
|implementing `Supplier>`; the function receives the raw request body, the path parameters and the CallContext.
|
+ |A `Query` body is a Dynamic Query declaration, not code: nothing is compiled, and it is checked against the Dynamic Entity definitions
+ |instead, with any problem reported in `errors` without a line number. A Dynamic Query only reads, so its `request_verb` must be `GET`,
+ |and it does not need user-supplied code to be enabled. See the Glossary entry Dynamic Query.
+ |
|`errors` carry the compiler's messages with `line` and `column` relative to the method body you sent (the server's added lines are
|subtracted; 0 when the compiler gave no position). When the body compiles and `dynamic_code_obp_calls_are_restricted` is on,
|the dependency validator runs too and any forbidden call is reported in `dependency_error`. `compiles` is true only when both pass.
@@ -6879,12 +6889,93 @@ object Http4s700 {
|${userAuthenticationMessage(true)}""".stripMargin,
JSONFactory700.dynamicResourceDocCompileJsonV700Example,
JSONFactory700.dynamicCompileResultJsonV700Example,
- List($AuthenticatedUserIsRequired, InvalidJsonFormat, UserHasMissingRoles, DynamicCodeExecutionDisabled, code.api.util.ErrorMessages.DynamicCodeLangNotSupport, code.api.util.ErrorMessages.TooManyRequests, UnknownError),
+ List($AuthenticatedUserIsRequired, InvalidJsonFormat, UserHasMissingRoles, DynamicCodeExecutionDisabled, code.api.util.ErrorMessages.DynamicCodeLangNotSupport,
+ code.api.util.ErrorMessages.DynamicQueryInvalid, code.api.util.ErrorMessages.TooManyRequests, UnknownError),
apiTagDynamicResourceDoc :: apiTagDynamic :: Nil,
Some(List(ApiRole.canCreateDynamicResourceDoc)),
http4sPartialFunction = Some(compileDynamicResourceDoc)
)
+ /**
+ * The value of a Right, or a failed Future carrying the Left's status and OBP error message, failed
+ * the way booleanToFuture fails one (an APIFailureNewStyle), so it is answered with that status.
+ */
+ private def answerOrFail[A](answer: Either[(Int, String), A], cc: CallContext): Future[A] = Future {
+ answer match {
+ case Right(value) => value
+ case Left((status, message)) =>
+ code.api.util.APIUtil.fullBoxOrException[A](
+ net.liftweb.common.Failure(message, net.liftweb.common.Empty, net.liftweb.common.Empty) ~> code.api.APIFailureNewStyle(message, status, Some(cc.toLight)))
+ .openOrThrowException(message)
+ }
+ }
+
+ // Route: POST /obp/v7.0.0/management/dynamic-resource-docs/explain
+ // For the author of a Dynamic Query: the statements it would run (SQL with ? for values) and the access
+ // it needs, for the requesting User or for an anonymous caller. Reads no record.
+ val explainDynamicQuery: HttpRoutes[IO] = HttpRoutes.of[IO] {
+ case req @ POST -> `prefixPath` / "management" / "dynamic-resource-docs" / "explain" =>
+ EndpointHelpers.withUser(req) { (u, cc) =>
+ import code.api.v7_0_0.JSONFactory700.{DynamicQueryExplainJsonV700, createDynamicQueryExplanationJsonV700}
+ import code.api.dynamic.entity.query.{DynamicQuery, DynamicQueryDeclaration}
+ for {
+ body <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the ${classOf[DynamicQueryExplainJsonV700].getSimpleName}", 400, Some(cc)) {
+ com.openbankproject.commons.util.JsonAliases.parse(cc.httpBody.getOrElse("")).extract[DynamicQueryExplainJsonV700]
+ }
+ declaration <- answerOrFail(
+ DynamicQueryDeclaration.parse(java.net.URLDecoder.decode(body.method_body, "UTF-8"))
+ .left.map(error => (400, s"${code.api.util.ErrorMessages.DynamicQueryInvalid}${error.message}")), cc)
+ space = body.bank_id.map(_.trim).filter(_.nonEmpty).flatMap(code.api.dynamic.entity.helper.DynamicEntitySpace.bankIdOrNoneForSystem)
+ callerUserId = if (body.as_anonymous_caller.contains(true)) None else Some(u.userId)
+ callerParameters = body.caller_parameters.map(_.trim).filter(_.nonEmpty)
+ .map(text => org.http4s.Query.unsafeFromString(text).multiParams.map { case (name, values) => name -> values.toList })
+ .getOrElse(Map.empty[String, List[String]])
+ explanation <- Future {
+ DynamicQuery.explain(space, declaration, callerParameters, callerUserId,
+ if (callerUserId.isEmpty) "" else code.api.util.APIUtil.getConsumerPrimaryKey(Some(cc)))
+ }
+ explained <- answerOrFail(explanation.left.map(failure => (failure.status, failure.message)), cc)
+ } yield createDynamicQueryExplanationJsonV700(explained, callerUserId)
+ }
+ }
+ resourceDocs += ResourceDoc(
+ implementedInApiVersion,
+ nameOf(explainDynamicQuery),
+ "POST",
+ "/management/dynamic-resource-docs/explain",
+ "Explain Dynamic Query",
+ s"""Explains how a Dynamic Query (a Dynamic Resource Doc whose `programming_lang` is `Query`) would be answered, without reading any record,
+ |so its author can check that the SQL is sane and that the access rules are the ones they expect.
+ |
+ |Send the URL-encoded declaration as `method_body`, as in a Dynamic Resource Doc. Optionally:
+ |
+ |* `bank_id`: the Dynamic Entity space to explain it in, a bank id or `SYS`. The system space when absent.
+ |* `caller_parameters`: the parameters a caller would add, as a query string, such as `obp_sort_by=name&obp_limit=10`.
+ |* `as_anonymous_caller`: true to explain it for a caller who is not logged in, rather than for you.
+ |
+ |The answer has two parts.
+ |
+ |`steps` are the reads the query would make, in order: the page, its count when the envelope names one, then each join. Each step
+ |says whether it goes through the query projection (with its SQL, every value shown as `?`), through the Dynamic Entity record
+ |provider (described in words: OBP does not build that SQL), or reuses an earlier step's records. The SQL comes from the same code
+ |that builds the statements a call runs, so it cannot differ from them.
+ |
+ |`entities`, `restricted_fields`, `rules`, `caller_may_run` and `refusal` describe access: every Dynamic Entity the query reads, with
+ |the Role that grants read access and whether the explained caller can read it; every read-restricted field it returns, copies,
+ |filters, sorts or picks by; the rules every Dynamic Query applies; and, when the caller could not run it, the exact refusal they
+ |would get (${code.api.util.ErrorMessages.DynamicQueryEntityNotReadable.takeWhile(_ != ':')} or ${code.api.util.ErrorMessages.DynamicQueryInvalid.takeWhile(_ != ':')}).
+ |
+ |A declaration that is not valid is answered with ${code.api.util.ErrorMessages.DynamicQueryInvalid.takeWhile(_ != ':')}, as Check would answer it. See the Glossary entry Dynamic Query.
+ |
+ |${userAuthenticationMessage(true)}""".stripMargin,
+ JSONFactory700.dynamicQueryExplainJsonV700Example,
+ JSONFactory700.dynamicQueryExplanationJsonV700Example,
+ List($AuthenticatedUserIsRequired, InvalidJsonFormat, UserHasMissingRoles, code.api.util.ErrorMessages.DynamicQueryInvalid, UnknownError),
+ apiTagDynamicResourceDoc :: apiTagDynamic :: Nil,
+ Some(List(ApiRole.canCreateDynamicResourceDoc)),
+ http4sPartialFunction = Some(explainDynamicQuery)
+ )
+
// Route: GET /obp/v7.0.0/management/dynamic-code-approval-config
// Lets a client (the API Manager create/edit pages) tell the maker up front whether a write will be
// applied or queued for approval. Authenticated, no role: any user who can create an artefact needs this.
diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala
index 80ca5fd32e..9d4712c53d 100644
--- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala
+++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala
@@ -1917,6 +1917,73 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats {
duration_ms = 850
)
+ // ─── Dynamic Query explain — the statements a Dynamic Query would run, and the access it needs ──
+
+ case class DynamicQueryExplainJsonV700(
+ // The declaration, URL-encoded as in a Dynamic Resource Doc's method_body.
+ method_body: String,
+ // The Dynamic Entity space to explain it in: a bank id, or SYS (the default) for the system space.
+ bank_id: Option[String] = None,
+ // Parameters a caller would add, as a query string, such as "obp_sort_by=name&obp_limit=10".
+ caller_parameters: Option[String] = None,
+ // Explain it for a caller who is not logged in, rather than for the User making this request.
+ as_anonymous_caller: Option[Boolean] = None
+ )
+ case class ExplainedEntityJsonV700(entity: String, read_role: String, bank_id: String, public_access: Boolean,
+ row_level_access: Boolean, caller_may_read: Boolean)
+ case class ExplainedFieldJsonV700(entity: String, field: String, restriction: String, read_role: String, caller_may_read: Boolean)
+ case class ExplainedStepJsonV700(step: Int, purpose: String, backend: String, sql: Option[String], parameter_count: Option[Int], notes: List[String])
+ case class DynamicQueryExplanationJsonV700(
+ space: String,
+ explained_for: String,
+ caller_may_run: Boolean,
+ refusal: Option[String],
+ entities: List[ExplainedEntityJsonV700],
+ restricted_fields: List[ExplainedFieldJsonV700],
+ rules: List[String],
+ steps: List[ExplainedStepJsonV700]
+ )
+
+ def createDynamicQueryExplanationJsonV700(explanation: code.api.dynamic.entity.query.DynamicQueryExplanation, callerUserId: Option[String]): DynamicQueryExplanationJsonV700 =
+ DynamicQueryExplanationJsonV700(
+ space = explanation.space,
+ explained_for = callerUserId.map(id => s"user $id").getOrElse("an anonymous caller"),
+ caller_may_run = explanation.callerMayRun,
+ refusal = explanation.refusal,
+ entities = explanation.entities.map(e => ExplainedEntityJsonV700(e.entity, e.readRole, e.bankId, e.publicAccess, e.rowLevelAccess, e.callerMayRead)),
+ restricted_fields = explanation.restrictedFields.map(f => ExplainedFieldJsonV700(f.entity, f.field, f.restriction, f.readRole, f.callerMayRead)),
+ rules = explanation.rules,
+ steps = explanation.steps.zipWithIndex.map { case (step, index) =>
+ ExplainedStepJsonV700(index + 1, step.purpose, step.backend, step.sql, step.sql.map(_.count(_ == '?')), step.notes)
+ }
+ )
+
+ lazy val dynamicQueryExplainJsonV700Example = DynamicQueryExplainJsonV700(
+ method_body = java.net.URLEncoder.encode(
+ """{"from":"activity","where":{"city":"eq:Berlin"},"join":[{"entity":"certificate","on":"activity_id","cardinality":"exists","as":"certified"}],"envelope":{"rows":"activities","count":"count"}}""",
+ "UTF-8"),
+ bank_id = None,
+ caller_parameters = Some("obp_sort_by=name&obp_limit=10"),
+ as_anonymous_caller = Some(false)
+ )
+ lazy val dynamicQueryExplanationJsonV700Example = DynamicQueryExplanationJsonV700(
+ space = "SYS",
+ explained_for = "user 9ca9a7e4-6d02-40e3-a129-0b2bf89de9b1",
+ caller_may_run = false,
+ refusal = Some("OBP-40066: This Dynamic Query reads Dynamic Entities you may not read: certificate (needs CanGetDynamicEntityRecord_certificate at bank SYS)."),
+ entities = List(
+ ExplainedEntityJsonV700("activity", "CanGetDynamicEntityRecord_activity", "SYS", public_access = true, row_level_access = false, caller_may_read = true),
+ ExplainedEntityJsonV700("certificate", "CanGetDynamicEntityRecord_certificate", "SYS", public_access = false, row_level_access = false, caller_may_read = false)),
+ restricted_fields = Nil,
+ rules = List("Only shared records are used, never a User's personal records, whoever owns them."),
+ steps = List(
+ ExplainedStepJsonV700(1, "Read the page of 'activity'", "projection",
+ Some("SELECT d.datajson FROM de_activity_ad1db27dae39 p JOIN dynamicdata d ON d.dynamicdataid = p.data_id WHERE d.dynamicentityname = ? AND d.bankid = ? AND d.ispersonalentity = ? AND p.c_city_11a62c23412b = CAST( ? AS text ) ORDER BY p.c_name_82a3537ff0db ASC LIMIT ?"),
+ Some(5), List("Filters and the sort run on the projection's indexed columns; only the records of the page are read.")),
+ ExplainedStepJsonV700(2, "Join 1: 'certificate' records whose 'activity_id' names the 'activity' (reverse)", "record provider", None, None,
+ List("Cardinality exists.")))
+ )
+
// ─── Dynamic code approval config — whether maker/checker gates dynamic artefacts on this instance ──
case class DynamicCodeApprovalConfigJsonV700(
diff --git a/obp-api/src/main/scala/code/dynamicEntity/DynamicEntityProvider.scala b/obp-api/src/main/scala/code/dynamicEntity/DynamicEntityProvider.scala
index 5638047b11..97323493c3 100644
--- a/obp-api/src/main/scala/code/dynamicEntity/DynamicEntityProvider.scala
+++ b/obp-api/src/main/scala/code/dynamicEntity/DynamicEntityProvider.scala
@@ -709,6 +709,10 @@ object DynamicEntityCommons extends Converter[DynamicEntityT, DynamicEntityCommo
if(readRoleRequired != JNothing) {
checkFormat(readRoleRequired.isInstanceOf[JBool], s"$DynamicEntityInstanceValidateFail The property of $fieldName's 'read_role_required' field must be boolean.")
}
+ val hideFromPublicAccess = value \ "hide_field_from_public_access"
+ if(hideFromPublicAccess != JNothing) {
+ checkFormat(hideFromPublicAccess.isInstanceOf[JBool], s"$DynamicEntityInstanceValidateFail The property of $fieldName's 'hide_field_from_public_access' field must be boolean.")
+ }
val writeRole = value \ "write_role"
if(writeRole != JNothing) {
checkFormat(writeRole.isInstanceOf[JString] && writeRole.asInstanceOf[JString].s.nonEmpty, s"$DynamicEntityInstanceValidateFail The property of $fieldName's 'write_role' field must be a non-empty string.")
diff --git a/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala b/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala
index 014308916b..ea7dabb871 100644
--- a/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala
+++ b/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala
@@ -365,7 +365,7 @@ object MakerChecker extends MdcLoggable {
for {
body <- parseAs[JsonDynamicResourceDoc](request.proposedPayload)
_ <- compileBox("dynamic resource doc") {
- val compiled = CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang)
+ val compiled = CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang, bankId)
compiled.validateDependency()
Full(compiled)
}
diff --git a/obp-api/src/main/scala/code/entitlement/Entilement.scala b/obp-api/src/main/scala/code/entitlement/Entilement.scala
index 3015135a16..f7141cd198 100644
--- a/obp-api/src/main/scala/code/entitlement/Entilement.scala
+++ b/obp-api/src/main/scala/code/entitlement/Entilement.scala
@@ -75,6 +75,8 @@ trait EntitlementProvider {
grantedByUserId: Option[String] = None,
groupId: Option[String] = None
): Box[Entitlement]
+ /** Delete the grants of a Dynamic Entity's record Roles in its own space only (bankId None is the
+ * system space); an entity of the same name in another space keeps its grants. */
def deleteDynamicEntityEntitlement(
entityName: String,
bankId: Option[String]
diff --git a/obp-api/src/main/scala/code/entitlement/MappedEntitlements.scala b/obp-api/src/main/scala/code/entitlement/MappedEntitlements.scala
index 830b4fbbee..adcbebc42e 100644
--- a/obp-api/src/main/scala/code/entitlement/MappedEntitlements.scala
+++ b/obp-api/src/main/scala/code/entitlement/MappedEntitlements.scala
@@ -27,7 +27,8 @@ TESOBE (http://www.tesobe.com/)
package code.entitlement
-import code.api.dynamic.entity.helper.DynamicEntityInfo
+import code.api.Constant.DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID
+import code.api.dynamic.entity.helper.{DynamicEntityInfo, DynamicEntitySpace}
import code.api.util.ApiRole.{
CanCreateEntitlementAtAnyBank,
CanCreateEntitlementAtOneBank
@@ -179,7 +180,20 @@ object MappedEntitlementsProvider extends EntitlementProvider with MdcLoggable {
bankId: Option[String]
): Box[Boolean] = {
val roleNames = DynamicEntityInfo.roleNames(entityName, bankId)
- deleteEntitlements(roleNames)
+ // A Role name carries no bank (CanGetDynamicEntityRecord_country), so the same names serve an
+ // entity of this name in every space: deleting by name alone would take every bank's grants with
+ // it. Only the grants of the deleted entity's space go: its bank id, or for the system space SYS
+ // and the empty bank id older grants were written at.
+ val bankIds = DynamicEntitySpace.bankIdOrNoneForSystem(bankId.getOrElse("")) match {
+ case Some(bank) => List(bank)
+ case None => List(DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID, "")
+ }
+ Box.tryo {
+ MappedEntitlement.bulkDelete_!!(
+ ByList(MappedEntitlement.mRoleName, roleNames),
+ ByList(MappedEntitlement.mBankId, bankIds)
+ )
+ }
}
override def deleteEntitlements(entityNames: List[String]): Box[Boolean] = {
diff --git a/obp-api/src/test/scala/code/api/ResourceDocs1_4_0/BankLevelDynamicResourceDocsTest.scala b/obp-api/src/test/scala/code/api/ResourceDocs1_4_0/BankLevelDynamicResourceDocsTest.scala
new file mode 100644
index 0000000000..3c335ed5ab
--- /dev/null
+++ b/obp-api/src/test/scala/code/api/ResourceDocs1_4_0/BankLevelDynamicResourceDocsTest.scala
@@ -0,0 +1,227 @@
+/**
+Open Bank Project - API
+Copyright (C) 2011-2026, TESOBE GmbH.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+ */
+package code.api.ResourceDocs1_4_0
+
+import code.api.Constant.DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID
+import code.api.util.APIUtil.OAuth._
+import code.api.util.ApiRole.canReadDynamicResourceDocsAtOneBank
+import code.api.util.ErrorMessages.{BankNotFound, UserHasMissingRoles}
+import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON
+import code.api.util.{ExampleValue, NewStyle}
+import code.dynamicEntity.{DynamicEntityCommons, DynamicEntityProvider, DynamicEntityT}
+import code.entitlement.Entitlement
+import code.setup.{DefaultUsers, PropsReset}
+import com.openbankproject.commons.util.ApiVersion
+import org.scalatest.Tag
+
+import scala.concurrent.Await
+import scala.concurrent.duration._
+
+/**
+ * This suite covers the bank level resource-docs routes, /banks/BANK_ID/resource-docs/API_VERSION/
+ * followed by obp, openapi or openapi.yaml.
+ *
+ * Each of them documents the dynamic things of one space only: the Dynamic Entities (and Dynamic
+ * Endpoints and Dynamic Resource Docs) of one bank, or of the system space when BANK_ID is SYS. The
+ * obp form used to ignore the bank after checking that it existed, and listed every space's dynamic
+ * docs, so every scenario here checks that the other spaces' entities are absent as well as that the
+ * requested space's entity is present.
+ */
+class BankLevelDynamicResourceDocsTest extends ResourceDocsV140ServerSetup with PropsReset with DefaultUsers {
+
+ object VersionOfApi extends Tag(ApiVersion.v1_4_0.toString)
+ object ObpFormat extends Tag("getBankLevelDynamicResourceDocsObp")
+ object OpenApiFormat extends Tag("getBankLevelDynamicResourceDocsOpenAPI31")
+
+ private val suffix = java.util.UUID.randomUUID().toString.replace("-", "").take(8)
+ private val bankOneEntity = s"rdbankone$suffix"
+ private val bankTwoEntity = s"rdbanktwo$suffix"
+ private val systemEntity = s"rdsystem$suffix"
+ private val allEntities = List(bankOneEntity, bankTwoEntity, systemEntity)
+
+ private val requestedVersion = ApiVersion.v7_0_0.toString
+ private def v7Request = baseRequest / "obp" / "v7.0.0"
+
+ private def definition(entity: String, space: Option[String]): DynamicEntityCommons =
+ DynamicEntityCommons(
+ entityName = entity,
+ metadataJson = s"""{"$entity":{"description":"An entity used by BankLevelDynamicResourceDocsTest.","required":[],"properties":{"name":{"type":"string","example":"Alice","description":"a name"}}}}""",
+ dynamicEntityId = None,
+ userId = resourceUser1.userId,
+ bankId = space,
+ hasPersonalEntity = false,
+ hasCommunityAccess = true
+ )
+
+ private def register(entity: String, space: Option[String]): DynamicEntityT =
+ DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(definition(entity, space))
+ .openOrThrowException(s"could not register $entity")
+
+ // The test setup empties every table before each scenario (LocalMappedConnectorTestSetup.wipeTestData),
+ // so the entities are registered again for each one.
+ override def beforeEach(): Unit = {
+ super.beforeEach()
+ register(bankOneEntity, Some(testBankId1.value))
+ register(bankTwoEntity, Some(testBankId2.value))
+ register(systemEntity, None)
+ }
+
+ /** Asserts the body names the one expected entity and none of the other spaces' entities. */
+ private def shouldDocumentOnly(body: String, expected: String): Unit = {
+ withClue(s"$expected should be documented. ") { body should include(expected) }
+ allEntities.filterNot(_ == expected).foreach { other =>
+ withClue(s"$other belongs to another space and should not be documented. ") { body should not include other }
+ }
+ }
+
+ feature("A bank level resource-docs document holds only that space's dynamic things") {
+
+ scenario("the obp format lists one bank's Dynamic Entities and no other space's", ObpFormat, VersionOfApi) {
+ val response = makeGetRequest((v7Request / "banks" / testBankId1.value / "resource-docs" / requestedVersion / "obp").GET)
+ response.code should equal(200)
+ shouldDocumentOnly(response.body.toString, bankOneEntity)
+ }
+
+ scenario("the openapi format lists one bank's Dynamic Entities and no other space's", OpenApiFormat, VersionOfApi) {
+ val response = makeGetRequest((v7Request / "banks" / testBankId2.value / "resource-docs" / requestedVersion / "openapi").GET)
+ response.code should equal(200)
+ (response.body \ "openapi").values.toString should startWith("3.1")
+ shouldDocumentOnly(response.body.toString, bankTwoEntity)
+ }
+
+ scenario("the openapi.yaml format at SYS lists the system level Dynamic Entities only", OpenApiFormat, VersionOfApi) {
+ val response = makeGetRequest((v7Request / "banks" / DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID / "resource-docs" / requestedVersion / "openapi.yaml").GET)
+ response.code should equal(200)
+ shouldDocumentOnly(response.body.toString, systemEntity)
+ }
+
+ scenario("the obp format at SYS lists the system level Dynamic Entities only", ObpFormat, VersionOfApi) {
+ val response = makeGetRequest((v7Request / "banks" / DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID / "resource-docs" / requestedVersion / "obp").GET)
+ response.code should equal(200)
+ shouldDocumentOnly(response.body.toString, systemEntity)
+ }
+ }
+
+ feature("A bank level resource-docs request is checked like the obp format always was") {
+
+ scenario("an unknown bank is a 404", OpenApiFormat, VersionOfApi) {
+ val response = makeGetRequest((v7Request / "banks" / s"no-such-bank-$suffix" / "resource-docs" / requestedVersion / "openapi").GET)
+ response.code should equal(404)
+ response.body.toString should include(BankNotFound)
+ }
+
+ scenario("an empty tags parameter is a 400", OpenApiFormat, VersionOfApi) {
+ val response = makeGetRequest((v7Request / "banks" / testBankId1.value / "resource-docs" / requestedVersion / "openapi").GET < "true")
+ val request = (v7Request / "banks" / DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID / "resource-docs" / requestedVersion / "openapi").GET <@ (user1)
+
+ Given("the caller holds the role at another bank only")
+ Entitlement.entitlement.vend.addEntitlement(testBankId1.value, resourceUser1.userId, canReadDynamicResourceDocsAtOneBank.toString)
+ val refused = makeGetRequest(request)
+ refused.code should equal(403)
+ refused.body.toString should include(UserHasMissingRoles)
+
+ Given("the caller holds the role at SYS")
+ Entitlement.entitlement.vend.addEntitlement(DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID, resourceUser1.userId, canReadDynamicResourceDocsAtOneBank.toString)
+ val allowed = makeGetRequest(request)
+ allowed.code should equal(200)
+ shouldDocumentOnly(allowed.body.toString, systemEntity)
+ }
+ }
+
+ feature("An instance wide content=dynamic document is rebuilt when a Dynamic Entity is added") {
+
+ // These documents used to be kept in the static swagger cache, which creating a Dynamic Entity
+ // does not clear, so a new entity stayed missing from them for the rest of the cache TTL.
+ scenario("the openapi content=dynamic document lists an entity created after it was first served", VersionOfApi) {
+ val request = (v7Request / "resource-docs" / requestedVersion / "openapi").GET <.
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+ */
+
+package code.api.dynamic.entity.query
+
+import org.json4s.JsonAST.{JBool, JString}
+import org.scalatest.{FlatSpec, Matchers}
+
+/**
+ * Pure unit tests for reading a Dynamic Query body: the shape it accepts, its defaults, and the
+ * messages for a malformed body. Checking a body against entity definitions, and running it, is
+ * covered by code.api.v4_0_0.DynamicQueryTest.
+ */
+class DynamicQueryDeclarationSpec extends FlatSpec with Matchers {
+
+ private def parsed(body: String): DynamicQueryDeclaration =
+ DynamicQueryDeclaration.parse(body).fold(e => fail(e.message), identity)
+ private def errorOf(body: String): String =
+ DynamicQueryDeclaration.parse(body).left.toOption.map(_.message).getOrElse(fail(s"expected an error for $body"))
+
+ "DynamicQueryDeclaration.parse" should "read every part of a full body" in {
+ val declaration = parsed(
+ """{
+ | "from": "activity",
+ | "select": ["activity_id", "name"],
+ | "where": { "city": "eq:Berlin", "status": ["ne:closed", "is_null"] },
+ | "join": [
+ | { "entity": "operator", "on": "operator_id", "fields": { "operator_legal_name": "legal_name" } },
+ | { "entity": "certificate", "on": "activity_id", "cardinality": "exists", "as": "certified",
+ | "where": { "status": "in:valid,renewed" }, "true_value": "yes", "false_value": false }
+ | ],
+ | "envelope": { "rows": "activities", "count": "count" }
+ |}""".stripMargin)
+ declaration.from shouldBe "activity"
+ declaration.select shouldBe Some(List("activity_id", "name"))
+ declaration.where shouldBe List(
+ Filter("city", FilterOp.Eq, List("Berlin")), Filter("status", FilterOp.Ne, List("closed")), Filter("status", FilterOp.IsNull, Nil))
+ declaration.joins.head shouldBe JoinRequest("operator", "operator_id", fields = List("operator_legal_name" -> "legal_name"))
+ val second = declaration.joins(1)
+ (second.cardinality, second.as, second.where) shouldBe ((Some("exists"), Some("certified"), List(Filter("status", FilterOp.In, List("valid", "renewed")))))
+ (second.trueValue, second.falseValue) shouldBe ((Some(JString("yes")), Some(JBool(false))))
+ declaration.envelope shouldBe DynamicQueryEnvelope("activities", Some("count"))
+ }
+
+ it should "default to all fields, no filters, no joins, and the entity's list name" in {
+ val declaration = parsed("""{ "from": "ActivityRecord" }""")
+ (declaration.select, declaration.where, declaration.joins) shouldBe ((None, Nil, Nil))
+ declaration.envelope shouldBe DynamicQueryEnvelope("activity_record_list", None)
+ }
+
+ it should "keep the order of a join's fields" in {
+ parsed("""{ "from": "a", "join": [ { "entity": "b", "on": "b_id", "fields": { "z": "f1", "a": "f2", "m": "f3" } } ] }""")
+ .joins.head.fields.map(_._1) shouldBe List("z", "a", "m")
+ }
+
+ it should "reject a body that is not a JSON object, or lacks 'from'" in {
+ errorOf("not json") should include("must be a JSON object")
+ errorOf("""["from"]""") should include("must be a JSON object")
+ errorOf("""{ "select": ["a"] }""") should include("needs 'from'")
+ }
+
+ it should "reject unknown keys, so a misspelt key is not silently ignored" in {
+ errorOf("""{ "from": "a", "joins": [] }""") should include("unknown key 'joins'")
+ errorOf("""{ "from": "a", "join": [ { "entity": "b", "on": "b_id", "feilds": {} } ] }""") should include("Join 1 has an unknown key 'feilds'")
+ errorOf("""{ "from": "a", "envelope": { "rows": "x", "total": "n" } }""") should include("unknown key 'total'")
+ }
+
+ it should "reject malformed parts with the place they are in" in {
+ errorOf("""{ "from": "a", "select": [] }""") should include("'select' must be a non-empty list")
+ errorOf("""{ "from": "a", "where": { "city": 3 } }""") should include("the filter on 'city' must be a string")
+ errorOf("""{ "from": "a", "where": { "city": "Berlin" } }""") should include(":")
+ errorOf("""{ "from": "a", "where": { "city": "near:Berlin" } }""") should include("Unknown filter operator 'near'")
+ errorOf("""{ "from": "a", "join": {} }""") should include("'join' must be a list")
+ errorOf("""{ "from": "a", "join": [ { "on": "b_id" } ] }""") should include("Join 1 needs 'entity'")
+ errorOf("""{ "from": "a", "join": [ { "entity": "b", "on": "b_id", "fields": { "x": 1 } } ] }""") should include("'x' does not")
+ errorOf("""{ "from": "a", "envelope": { "rows": "n", "count": "n" } }""") should include("one name for both")
+ }
+}
diff --git a/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinSpec.scala b/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinSpec.scala
new file mode 100644
index 0000000000..bbf41d8243
--- /dev/null
+++ b/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinSpec.scala
@@ -0,0 +1,244 @@
+/**
+Open Bank Project - API
+Copyright (C) 2011-2026, TESOBE GmbH.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+ */
+
+package code.api.dynamic.entity.query
+
+import com.openbankproject.commons.model.enums.DynamicEntityFieldType
+import org.json4s.JsonAST._
+import org.scalatest.{FlatSpec, Matchers}
+
+/**
+ * Pure unit tests for joins: the planner's checks, the direction it infers, and the merge of records
+ * already read into a page. No server and no database; reading the other records, with personal
+ * records, row-level access and the projection, is covered by
+ * code.api.v6_0_0.DynamicEntityJoinPlanIntegrationTest.
+ *
+ * Domain:
+ * - `activity` (name, city, operator_id : reference:operator);
+ * - `operator` (legal_name, country, status);
+ * - `certificate` (number, status, issue_date, secret, activity_id : reference:activity indexed,
+ * loose_activity_id : reference:activity not indexed);
+ * - `employee` (name, manager_id : reference:employee indexed), a self-reference.
+ */
+class JoinSpec extends FlatSpec with Matchers {
+
+ import DynamicEntityFieldType._
+
+ private val activity = JoinEntityInfo(Set("name", "city", "operator_id"), "activity_id",
+ Map("operator_id" -> "operator"), Map("name" -> string, "city" -> string, "operator_id" -> reference))
+ private val operator = JoinEntityInfo(Set("legal_name", "country", "status"), "operator_id", Map.empty,
+ Map("legal_name" -> string, "country" -> string, "status" -> string))
+ private val certificate = JoinEntityInfo(
+ Set("number", "status", "issue_date", "secret", "activity_id", "loose_activity_id"), "certificate_id",
+ Map("activity_id" -> "activity", "loose_activity_id" -> "activity"),
+ Map("number" -> string, "status" -> string, "issue_date" -> DATE_WITH_DAY, "secret" -> string,
+ "activity_id" -> reference, "loose_activity_id" -> reference),
+ Set("activity_id"))
+ private val employee = JoinEntityInfo(Set("name", "manager_id"), "employee_id", Map("manager_id" -> "employee"),
+ Map("name" -> string, "manager_id" -> reference), Set("manager_id"))
+ private val entities = Map("activity" -> activity, "operator" -> operator, "certificate" -> certificate, "employee" -> employee)
+
+ private def planOf(parent: String, mayReadEntity: String => Boolean, mayReadField: (String, String) => Boolean)(requests: JoinRequest*) =
+ JoinPlanner.plan(parent, entities(parent), requests.toList, entities.get, mayReadEntity, mayReadField)
+ private def plan(requests: JoinRequest*) = planOf("activity", _ => true, (_, _) => true)(requests: _*)
+ private def errorOf(result: Either[QueryError, List[Join]]): String =
+ result.left.toOption.map(_.message).getOrElse(fail(s"expected an error, got $result"))
+ private def planned(requests: JoinRequest*): List[Join] = plan(requests: _*).fold(e => fail(e.message), identity)
+
+ private def operatorJoin(fields: (String, String)*) = JoinRequest("operator", "operator_id", fields = fields.toList)
+ private def latestCertificate(fields: (String, String)*) =
+ JoinRequest("certificate", "activity_id", cardinality = Some("at_most_one"), pick = Some("latest_by:issue_date"), fields = fields.toList)
+
+ // ----- direction -----
+
+ "JoinPlanner" should "infer forward when the parent holds the reference, at_most_one by default" in {
+ val join = planned(operatorJoin("operator_legal_name" -> "legal_name")).head
+ (join.direction, join.cardinality) shouldBe ((JoinDirection.Forward, Cardinality.AtMostOne))
+ }
+
+ it should "infer reverse when the other entity holds the reference" in {
+ planned(latestCertificate("n" -> "number")).head.direction shouldBe JoinDirection.Reverse
+ }
+
+ it should "require a direction for a self-reference, and honour it" in {
+ def employeePlan(requests: JoinRequest*) = planOf("employee", _ => true, (_, _) => true)(requests: _*)
+ errorOf(employeePlan(JoinRequest("employee", "manager_id", fields = List("manager_name" -> "name")))) should include("Say which with \"direction\"")
+ employeePlan(
+ JoinRequest("employee", "manager_id", direction = Some("forward"), fields = List("manager_name" -> "name")),
+ JoinRequest("employee", "manager_id", direction = Some("reverse"), cardinality = Some("many"), as = Some("reports"), fields = List("name" -> "name"))
+ ).map(_.map(_.direction)) shouldBe Right(List(JoinDirection.Forward, JoinDirection.Reverse))
+ }
+
+ it should "reject a direction the definitions do not support, or an unknown one" in {
+ errorOf(plan(operatorJoin("x" -> "legal_name").copy(direction = Some("reverse")))) should include("direction is reverse, but 'operator' has no field 'operator_id' typed 'reference:activity'")
+ errorOf(plan(operatorJoin("x" -> "legal_name").copy(direction = Some("sideways")))) should include("direction must be 'forward' or 'reverse'")
+ }
+
+ it should "reject an 'on' field that links the two in neither direction" in {
+ errorOf(plan(JoinRequest("operator", "city", fields = List("x" -> "legal_name")))) should include(
+ "'activity' has no field 'city' typed 'reference:operator' and 'operator' has no field 'city' typed 'reference:activity'")
+ }
+
+ // ----- other checks -----
+
+ it should "reject an unknown entity, or one the caller may not read" in {
+ errorOf(plan(JoinRequest("permit", "activity_id", cardinality = Some("exists"), as = Some("x")))) should include("no Dynamic Entity 'permit'")
+ errorOf(planOf("activity", _ != "operator", (_, _) => true)(operatorJoin("x" -> "legal_name"))) should include("you may not read 'operator'")
+ }
+
+ it should "require a reverse join's link field to be indexed, but not a forward one's" in {
+ errorOf(plan(JoinRequest("certificate", "loose_activity_id", cardinality = Some("exists"), as = Some("x")))) should include("must be declared \"indexed\": true")
+ plan(operatorJoin("x" -> "legal_name")).isRight shouldBe true // activity.operator_id is not indexed
+ }
+
+ it should "fit the cardinality to the direction" in {
+ errorOf(plan(operatorJoin("x" -> "legal_name").copy(cardinality = Some("many"), as = Some("y")))) should include("cannot be 'many'")
+ errorOf(plan(JoinRequest("certificate", "activity_id", as = Some("x")))) should include("needs \"cardinality\"")
+ errorOf(plan(JoinRequest("certificate", "activity_id", cardinality = Some("several"), as = Some("x")))) should include("cardinality must be one of")
+ plan(operatorJoin().copy(cardinality = Some("exists"), as = Some("has_operator"))).isRight shouldBe true
+ }
+
+ it should "require a pick for a reverse at_most_one, and refuse one for a forward join" in {
+ errorOf(plan(latestCertificate("n" -> "number").copy(pick = None))) should include("at_most_one needs 'pick'")
+ errorOf(plan(operatorJoin("x" -> "legal_name").copy(pick = Some("latest_by:legal_name")))) should include("'pick' is not used by a join that can find at most one record")
+ errorOf(plan(latestCertificate("n" -> "number").copy(pick = Some("newest:issue_date")))) should include("must be 'latest_by:' or 'earliest_by:'")
+ errorOf(plan(latestCertificate("n" -> "number").copy(pick = Some("latest_by:missing")))) should include("names 'missing'")
+ }
+
+ it should "require the shape each cardinality needs" in {
+ errorOf(plan(operatorJoin())) should include("at_most_one needs 'fields'")
+ errorOf(plan(operatorJoin("x" -> "legal_name").copy(as = Some("y")))) should include("'as' is not used with cardinality at_most_one")
+ errorOf(plan(JoinRequest("certificate", "activity_id", cardinality = Some("many"), fields = List("n" -> "number")))) should include("many needs 'as'")
+ errorOf(plan(JoinRequest("certificate", "activity_id", cardinality = Some("many"), as = Some("x")))) should include("many needs 'fields'")
+ errorOf(plan(JoinRequest("certificate", "activity_id", cardinality = Some("exists"), as = Some("x"), fields = List("n" -> "number")))) should include("'fields' is not used")
+ errorOf(plan(operatorJoin("x" -> "no_such_field"))) should include("'operator' has no field 'no_such_field'")
+ }
+
+ it should "reject a result name taken by the parent, an earlier join, used twice, or unusable" in {
+ errorOf(plan(operatorJoin("name" -> "legal_name"))) should include("'name' is already a field of the result")
+ errorOf(plan(operatorJoin("activity_id" -> "legal_name"))) should include("'activity_id' is already a field of the result")
+ errorOf(plan(operatorJoin("x" -> "legal_name"), latestCertificate("x" -> "number"))) should include("'x' is already a field of the result")
+ errorOf(plan(operatorJoin("x" -> "legal_name", "x" -> "country"))) should include("a result name is used twice")
+ errorOf(plan(operatorJoin("1st" -> "legal_name"))) should include("must be a field name")
+ }
+
+ it should "validate where filters with the list endpoint's rules, in either direction" in {
+ def whereOn(filter: Filter) = JoinRequest("certificate", "activity_id", cardinality = Some("exists"), as = Some("x"), where = List(filter))
+ errorOf(plan(whereOn(Filter("issue_date", FilterOp.Eq, List("not-a-date"))))) should include("not a valid 'DATE_WITH_DAY'")
+ errorOf(plan(whereOn(Filter("status", FilterOp.Lt, List("a"))))) should include("Operator 'lt' is not valid")
+ errorOf(plan(whereOn(Filter("nope", FilterOp.Eq, List("a"))))) should include("names 'nope'")
+ plan(operatorJoin("x" -> "legal_name").copy(where = List(Filter("status", FilterOp.Eq, List("active"))))).isRight shouldBe true
+ }
+
+ it should "refuse to filter or order by a field the caller may not read" in {
+ def noSecret(request: JoinRequest) = planOf("activity", _ => true, (_, field) => field != "secret")(request)
+ errorOf(noSecret(JoinRequest("certificate", "activity_id", cardinality = Some("exists"), as = Some("x"),
+ where = List(Filter("secret", FilterOp.Eq, List("a")))))) should include("you may not read 'secret'")
+ errorOf(noSecret(latestCertificate("n" -> "number").copy(pick = Some("latest_by:secret")))) should include("you may not read 'secret'")
+ }
+
+ // ----- merge -----
+
+ private def cert(id: String, number: String, status: String, issueDate: Option[String], activityId: String): JObject =
+ JObject(List(JField("certificate_id", JString(id)), JField("number", JString(number)), JField("status", JString(status)),
+ JField("activity_id", JString(activityId))) ++ issueDate.map(d => JField("issue_date", JString(d))).toList)
+ private def activityRecord(id: String, operatorId: JValue): JObject =
+ JObject(JField("activity_id", JString(id)), JField("operator_id", operatorId))
+
+ private val linked: RecordJoiner.LinkedRecords = Map(
+ RecordJoiner.Link("operator", "operator_id", JoinDirection.Forward) -> Map(
+ "op-1" -> List(JObject(JField("operator_id", JString("op-1")), JField("legal_name", JString("Acme Ltd")), JField("country", JString("DE")), JField("status", JString("active")))),
+ "op-2" -> List(JObject(JField("operator_id", JString("op-2")), JField("legal_name", JString("No Country Ltd")), JField("status", JString("closed"))))),
+ RecordJoiner.Link("certificate", "activity_id", JoinDirection.Reverse) -> Map("a1" -> List(
+ cert("c3", "C-3", "valid", None, "a1"),
+ cert("c2", "C-2", "revoked", Some("2026-06-01"), "a1"),
+ cert("c1", "C-1", "valid", Some("2026-01-01"), "a1"),
+ cert("c0", "C-0", "valid", Some("2026-06-01"), "a1"))))
+
+ private val page = List(
+ activityRecord("a1", JString("op-1")),
+ activityRecord("a2", JString("op-2")),
+ activityRecord("a3", JString("op-missing")),
+ activityRecord("a4", JNull))
+
+ private def mergeWith(readable: (String, String) => Boolean)(requests: JoinRequest*): List[JObject] =
+ RecordJoiner.merge(page, planned(requests: _*), "activity_id", linked, readable)
+ private def merge(requests: JoinRequest*): List[JObject] = mergeWith((_, _) => true)(requests: _*)
+
+ "RecordJoiner.merge" should "copy a forward join's fields, null when there is no record or no field" in {
+ val merged = merge(operatorJoin("operator_legal_name" -> "legal_name", "operator_country" -> "country"))
+ merged.map(_ \ "operator_legal_name") shouldBe List(JString("Acme Ltd"), JString("No Country Ltd"), JNull, JNull)
+ merged.map(_ \ "operator_country") shouldBe List(JString("DE"), JNull, JNull, JNull)
+ merged.head.obj.map(_._1) shouldBe List("activity_id", "operator_id", "operator_legal_name", "operator_country")
+ }
+
+ it should "apply where to a forward join, and answer exists for one" in {
+ merge(operatorJoin("active_operator" -> "legal_name").copy(where = List(Filter("status", FilterOp.Eq, List("active")))))
+ .map(_ \ "active_operator") shouldBe List(JString("Acme Ltd"), JNull, JNull, JNull)
+ merge(operatorJoin().copy(cardinality = Some("exists"), as = Some("has_operator")))
+ .map(_ \ "has_operator") shouldBe List(JBool(true), JBool(true), JBool(false), JBool(false))
+ }
+
+ it should "pick the latest in a reverse join, breaking a tie by record id, never choosing a record without the date" in {
+ merge(latestCertificate("n" -> "number")).map(_ \ "n") shouldBe List(JString("C-0"), JNull, JNull, JNull)
+ }
+
+ it should "pick the earliest, still putting a record without the date last" in {
+ merge(latestCertificate("n" -> "number").copy(pick = Some("earliest_by:issue_date"))).head \ "n" shouldBe JString("C-1")
+ }
+
+ it should "apply where before the pick" in {
+ merge(latestCertificate("n" -> "number").copy(where = List(Filter("status", FilterOp.Eq, List("revoked"))))).head \ "n" shouldBe JString("C-2")
+ }
+
+ it should "list many in order, by record id without one, and give an empty array to a record with none" in {
+ val ordered = merge(JoinRequest("certificate", "activity_id", cardinality = Some("many"), as = Some("all"),
+ order = Some("earliest_by:issue_date"), fields = List("n" -> "number")))
+ ordered.head \ "all" \ "n" shouldBe JArray(List(JString("C-1"), JString("C-0"), JString("C-2"), JString("C-3")))
+ ordered(1) \ "all" shouldBe JArray(Nil)
+ merge(JoinRequest("certificate", "activity_id", cardinality = Some("many"), as = Some("all"), fields = List("n" -> "number")))
+ .head \ "all" \ "n" shouldBe JArray(List(JString("C-0"), JString("C-1"), JString("C-2"), JString("C-3")))
+ }
+
+ it should "give exists its values, true and false by default" in {
+ merge(JoinRequest("certificate", "activity_id", cardinality = Some("exists"), as = Some("certified")))
+ .map(_ \ "certified") shouldBe List(JBool(true), JBool(false), JBool(false), JBool(false))
+ merge(JoinRequest("certificate", "activity_id", cardinality = Some("exists"), as = Some("state"),
+ where = List(Filter("status", FilterOp.Eq, List("revoked"))), trueValue = Some(JString("revoked")), falseValue = Some(JString("clean"))))
+ .map(_ \ "state") shouldBe List(JString("revoked"), JString("clean"), JString("clean"), JString("clean"))
+ }
+
+ it should "give null for a copied field the caller may not read, without hiding the others" in {
+ val merged = mergeWith((_, field) => field != "country")(operatorJoin("name1" -> "legal_name", "c" -> "country"))
+ (merged.head \ "name1", merged.head \ "c") shouldBe ((JString("Acme Ltd"), JNull))
+ }
+
+ it should "never add or remove a record, or change their order" in {
+ merge(operatorJoin("x" -> "legal_name"), latestCertificate("n" -> "number")).map(_ \ "activity_id") shouldBe
+ List(JString("a1"), JString("a2"), JString("a3"), JString("a4"))
+ }
+}
diff --git a/obp-api/src/test/scala/code/api/util/SelfServiceRateLimiterTest.scala b/obp-api/src/test/scala/code/api/util/SelfServiceRateLimiterTest.scala
index 7ab645902b..3638804b57 100644
--- a/obp-api/src/test/scala/code/api/util/SelfServiceRateLimiterTest.scala
+++ b/obp-api/src/test/scala/code/api/util/SelfServiceRateLimiterTest.scala
@@ -245,6 +245,8 @@ class SelfServiceRateLimiterTest extends ServerSetup {
SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/resource-docs/v6.0.0/openapi")) shouldBe Some("documentation")
SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/resource-docs/v6.0.0/openapi.yaml")) shouldBe Some("documentation")
SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v4.0.0/banks/gh.29.uk/resource-docs/v4.0.0/obp")) shouldBe Some("documentation")
+ SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v7.0.0/banks/SYS/resource-docs/v7.0.0/openapi")) shouldBe Some("documentation")
+ SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v7.0.0/banks/gh.29.uk/resource-docs/OBPv7.0.0/openapi.yaml")) shouldBe Some("documentation")
SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v5.1.0/message-docs/rest_vMar2019/swagger2.0")) shouldBe Some("documentation")
SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v2.2.0/message-docs/rest_vMar2019")) shouldBe Some("documentation")
SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/message-docs/rest_vMar2019/json-schema")) shouldBe Some("documentation")
diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicEntityTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicEntityTest.scala
index 4200fbb565..71e6568b06 100644
--- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicEntityTest.scala
+++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicEntityTest.scala
@@ -1680,14 +1680,6 @@ class DynamicEntityTest extends V400ServerSetup {
typeResponse.code should equal(400)
typeResponse.body.extract[ErrorMessage].message should include (DynamicEntityUpdateNotSchemaCompatible)
- Then("adding a property is refused")
- val propertyAdded = rightEntity.transformField { case JField("properties", JObject(fields)) =>
- JField("properties", JObject(fields :+ JField("colour", JObject(List(JField("type", JString("string")), JField("example", JString("red")))))))
- }
- val addResponse = makePutRequest(updateRequest, write(propertyAdded))
- addResponse.code should equal(400)
- addResponse.body.extract[ErrorMessage].message should include (DynamicEntityUpdateNotSchemaCompatible)
-
Then("making an existing optional property required is refused")
val requiredGrown = rightEntity.transformField { case JField("required", JArray(items)) =>
JField("required", JArray(items :+ JString("number")))
@@ -1695,6 +1687,31 @@ class DynamicEntityTest extends V400ServerSetup {
val requiredResponse = makePutRequest(updateRequest, write(requiredGrown))
requiredResponse.code should equal(400)
requiredResponse.body.extract[ErrorMessage].message should include (DynamicEntityUpdateNotSchemaCompatible)
+
+ val colour = JField("colour", JObject(List(JField("type", JString("string")), JField("example", JString("red")))))
+ def withColour(entity: JValue): JValue = entity.transformField { case JField("properties", JObject(fields)) =>
+ JField("properties", JObject(fields :+ colour))
+ }
+
+ Then("adding a required property is refused: the stored record lacks it")
+ val requiredAdded = withColour(rightEntity).transformField { case JField("required", JArray(items)) =>
+ JField("required", JArray(items :+ JString("colour")))
+ }
+ val requiredAddedResponse = makePutRequest(updateRequest, write(requiredAdded))
+ requiredAddedResponse.code should equal(400)
+ requiredAddedResponse.body.extract[ErrorMessage].message should include (DynamicEntityUpdateNotSchemaCompatible)
+
+ Then("adding an optional property is accepted, and the record is kept")
+ val addResponse = makePutRequest(updateRequest, write(withColour(rightEntity)))
+ addResponse.code should equal(200)
+ (addResponse.body \ "FooBar" \ "properties" \ "colour" \ "type") should equal(JString("string"))
+ val afterAdd = makeGetRequest((dynamicEntity_Request / "FooBar").GET <@(user1))
+ (afterAdd.body \ "foo_bar_list").asInstanceOf[JArray].arr.size should equal(1)
+
+ Then("removing that property again is refused")
+ val removeResponse = makePutRequest(updateRequest, write(rightEntity))
+ removeResponse.code should equal(400)
+ removeResponse.body.extract[ErrorMessage].message should include (DynamicEntityUpdateNotSchemaCompatible)
}
}
diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicQueryAccessTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicQueryAccessTest.scala
new file mode 100644
index 0000000000..eb039b0bc4
--- /dev/null
+++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicQueryAccessTest.scala
@@ -0,0 +1,396 @@
+package code.api.v4_0_0
+
+import code.DynamicData.{DynamicDataAccessProvider, DynamicDataProvider}
+import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON
+import code.api.dynamic.entity.helper.DynamicEntitySpace
+import code.api.util.APIUtil.OAuth._
+import code.api.util.ApiRole
+import cats.effect.unsafe.implicits.global
+import code.api.dynamic.entity.projection.{IndexingCapabilities, ProjectionProvisioner}
+import code.api.util.ErrorMessages.{DynamicEntityFieldNotReadable, DynamicQueryEntityNotReadable, DynamicQueryInvalid, UserHasMissingRoles}
+import code.dynamicEntity.{DynamicEntityCommons, DynamicEntityProvider}
+import code.entitlement.Entitlement
+import code.setup.OBPReq
+import com.openbankproject.commons.model.ErrorMessage
+import net.liftweb.util.StringHelpers
+import org.json4s._
+import org.json4s.native.Serialization.write
+
+import java.net.URLEncoder
+
+/**
+ * This suite checks who can read what through one Dynamic Query that joins four Dynamic Entities,
+ * end to end over HTTP, as the entities' access and the caller's Roles vary.
+ *
+ * The four entities, created afresh for each scenario:
+ * - `activity`, the query's `from`;
+ * - `operator`, joined forward through `activity.operator_id`; its `tax_number` needs its own read Role;
+ * - `country`, joined forward through `activity.country_id`;
+ * - `certificate`, joined in reverse through `certificate.activity_id`, twice: the latest one, and
+ * whether any exists.
+ *
+ * The rule under test: the entities' own access is the primary control. A caller must be able to read
+ * every entity the query reads (public access, the entity's read Role, or row-level access); Roles on
+ * the doc itself can only narrow who may call it, never widen what a caller can read.
+ */
+class DynamicQueryAccessTest extends V400ServerSetup {
+
+ private val owner = "dynamic-query-access-owner"
+ private def idField(entity: String): String = StringHelpers.snakify(entity) + "_id"
+
+ /** The four entities of one scenario, and the records the assertions refer to. */
+ private case class Registry(activity: String, operator: String, country: String, certificate: String,
+ withEverything: String, withNothing: String, acme: String)
+
+ private def createDef(entity: String, propsJson: String, publicAccess: Boolean, rowLevel: Boolean = false): Unit =
+ DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(
+ DynamicEntityCommons(entity, s"""{"$entity":{"properties":$propsJson}}""", None, owner, None,
+ hasPersonalEntity = true, hasPublicAccess = publicAccess, useRowLevelAccess = rowLevel)
+ ).openOrThrowException(s"failed to create definition for $entity")
+
+ private def saveRec(entity: String, fields: (String, JValue)*): String = saveAs(entity, None, fields: _*)
+
+ /** Save a record, shared unless `personalOwner` is given. Returns its id. */
+ private def saveAs(entity: String, personalOwner: Option[String], fields: (String, JValue)*): String = {
+ val id = java.util.UUID.randomUUID().toString
+ val body = JObject(JField(idField(entity), JString(id)) :: fields.toList.map { case (k, v) => JField(k, v) })
+ DynamicDataProvider.connectorMethodProvider.vend.save(None, entity, body, personalOwner.orElse(Some(owner)), personalOwner.isDefined)
+ .openOrThrowException(s"failed to save $entity record")
+ id
+ }
+
+ /** Four entities; `public` names the ones with public access (by role in the query: activity, operator, country, certificate). */
+ private def registry(public: Set[String]): Registry = {
+ val sfx = java.util.UUID.randomUUID().toString.take(8).replace("-", "")
+ val r = Registry(s"Activity$sfx", s"Operator$sfx", s"Country$sfx", s"Certificate$sfx", "", "", "")
+ createDef(r.operator, s"""{"${idField(r.operator)}":{"type":"string"},"legal_name":{"type":"string"},""" +
+ s""""tax_number":{"type":"string","read_role_required":true},"contact_email":{"type":"string","hide_field_from_public_access":true}}""",
+ public.contains("operator"))
+ createDef(r.country, s"""{"${idField(r.country)}":{"type":"string"},"name":{"type":"string"}}""", public.contains("country"))
+ createDef(r.activity, s"""{"${idField(r.activity)}":{"type":"string"},"name":{"type":"string","indexed":true},""" +
+ s""""operator_id":{"type":"reference:${r.operator}"},"country_id":{"type":"reference:${r.country}"}}""", public.contains("activity"))
+ createDef(r.certificate, s"""{"${idField(r.certificate)}":{"type":"string"},"number":{"type":"string"},"issue_date":{"type":"DATE_WITH_DAY"},""" +
+ s""""activity_id":{"type":"reference:${r.activity}","indexed":true}}""", public.contains("certificate"))
+
+ val acme = saveRec(r.operator, "legal_name" -> JString("Acme Ltd"), "tax_number" -> JString("DE-123"),
+ "contact_email" -> JString("office@acme.example"))
+ val germany = saveRec(r.country, "name" -> JString("Germany"))
+ val withEverything = saveRec(r.activity, "name" -> JString("1 with everything"), "operator_id" -> JString(acme), "country_id" -> JString(germany))
+ val withNothing = saveRec(r.activity, "name" -> JString("2 with nothing"))
+ saveRec(r.certificate, "number" -> JString("C-old"), "issue_date" -> JString("2026-01-01"), "activity_id" -> JString(withEverything))
+ saveRec(r.certificate, "number" -> JString("C-new"), "issue_date" -> JString("2026-06-01"), "activity_id" -> JString(withEverything))
+ r.copy(withEverything = withEverything, withNothing = withNothing, acme = acme)
+ }
+
+ private def declaration(r: Registry, extraJoins: String = ""): String =
+ s"""{
+ | "from": "${r.activity}",
+ | "select": ["name"],
+ | "join": [
+ | { "entity": "${r.operator}", "on": "operator_id", "fields": { "operator_name": "legal_name", "operator_tax_number": "tax_number" } },
+ | { "entity": "${r.country}", "on": "country_id", "fields": { "country_name": "name" } },
+ | { "entity": "${r.certificate}", "on": "activity_id", "cardinality": "at_most_one", "pick": "latest_by:issue_date",
+ | "fields": { "latest_certificate": "number" } },
+ | { "entity": "${r.certificate}", "on": "activity_id", "cardinality": "exists", "as": "certified" }$extraJoins
+ | ],
+ | "envelope": { "rows": "activities", "count": "count" }
+ |}""".stripMargin
+
+ /** Create a Dynamic Query doc as user1 (who holds the create Role) and return the path it is served at. */
+ private def createQuery(segment: String, body: String, roles: String = "") = {
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString)
+ val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy(
+ dynamicResourceDocId = None, bankId = None, roles = roles,
+ partialFunctionName = s"accessQuery${segment.capitalize}", requestVerb = "GET", requestUrl = s"/$segment",
+ exampleRequestBody = None, errorResponseBodies = "OBP-50000: Unknown Error.",
+ methodBody = URLEncoder.encode(body, "UTF-8"), programmingLang = "Query")
+ val created = makePostRequest((v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1), write(doc))
+ withClue(s"create: ${created.body}") { created.code should equal(201) }
+ dynamicEndpoint_Request / "dynamic-resource-doc" / segment
+ }
+
+ private def grantRead(userId: String, entity: String): Unit =
+ Entitlement.entitlement.vend.addEntitlement(DynamicEntitySpace.bankIdOrSystem(None), userId, s"CanGetDynamicEntityRecord_$entity")
+
+ private def needs(entity: String) = s"$entity (needs CanGetDynamicEntityRecord_$entity at bank SYS)"
+ private def notReadable(entities: String*) = s"$DynamicQueryEntityNotReadable${entities.map(needs).mkString(", ")}."
+ private def messageOf(response: code.setup.APIResponse): String = response.body.extract[ErrorMessage].message
+
+ /** The rows of an answer, by activity name. */
+ private def rows(response: code.setup.APIResponse): Map[String, JValue] =
+ withClue(s"response: ${response.body}") {
+ response.code should equal(200)
+ (response.body \ "activities").asInstanceOf[JArray].arr.map(row => (row \ "name").values.toString -> row).toMap
+ }
+
+ private val everything = "1 with everything"
+ private val nothing = "2 with nothing"
+
+ feature("A Dynamic Query that joins four Dynamic Entities, as their access and the caller's Roles vary") {
+
+ scenario("all four entities public, no Roles on the doc: anyone can call it, and a read-restricted field stays null") {
+ val r = registry(public = Set("activity", "operator", "country", "certificate"))
+ val call = createQuery(s"access_all_public_${r.activity}", declaration(r))
+
+ val answered = rows(makeGetRequest(call))
+ answered(everything) \ "operator_name" shouldBe JString("Acme Ltd")
+ answered(everything) \ "country_name" shouldBe JString("Germany")
+ answered(everything) \ "latest_certificate" shouldBe JString("C-new")
+ answered(everything) \ "certified" shouldBe JBool(true)
+ answered(everything) \ "operator_tax_number" shouldBe JNull
+ answered(nothing) \ "operator_name" shouldBe JNull
+ answered(nothing) \ "certified" shouldBe JBool(false)
+ }
+
+ scenario("some entities public, some not: each caller is told every entity they may not read, until they may read them all") {
+ val r = registry(public = Set("activity", "country"))
+ val call = createQuery(s"access_mixed_${r.activity}", declaration(r))
+
+ Then("an anonymous caller is refused, naming both entities that are not public, in the query's order")
+ val anonymous = makeGetRequest(call)
+ anonymous.code should equal(403)
+ messageOf(anonymous) shouldBe notReadable(r.operator, r.certificate)
+
+ And("so is a logged-in caller with no Roles")
+ messageOf(makeGetRequest(call <@ (user2))) shouldBe notReadable(r.operator, r.certificate)
+
+ When("the caller may read the operators")
+ grantRead(resourceUser2.userId, r.operator)
+ Then("only the certificates are still named")
+ messageOf(makeGetRequest(call <@ (user2))) shouldBe notReadable(r.certificate)
+
+ When("the caller may read the certificates too")
+ grantRead(resourceUser2.userId, r.certificate)
+ val answered = rows(makeGetRequest(call <@ (user2)))
+ answered(everything) \ "operator_name" shouldBe JString("Acme Ltd")
+ answered(everything) \ "latest_certificate" shouldBe JString("C-new")
+ (makeGetRequest(call <@ (user2)).body \ "count") shouldBe JInt(2)
+
+ And("another caller without those Roles is still refused")
+ messageOf(makeGetRequest(call <@ (user3))) shouldBe notReadable(r.operator, r.certificate)
+ }
+
+ scenario("no entity public: the caller needs every entity's read Role, including the one it reads from") {
+ val r = registry(public = Set.empty)
+ val call = createQuery(s"access_none_public_${r.activity}", declaration(r))
+ messageOf(makeGetRequest(call <@ (user2))) shouldBe notReadable(r.activity, r.operator, r.country, r.certificate)
+ List(r.activity, r.operator, r.country).foreach(grantRead(resourceUser2.userId, _))
+ messageOf(makeGetRequest(call <@ (user2))) shouldBe notReadable(r.certificate)
+ grantRead(resourceUser2.userId, r.certificate)
+ rows(makeGetRequest(call <@ (user2)))(everything) \ "country_name" shouldBe JString("Germany")
+ }
+
+ scenario("a Role on the doc only narrows who may call it; it never stands in for access to an entity") {
+ val r = registry(public = Set("activity", "country"))
+ val docRole = s"CanReadRegistry${r.activity}"
+ val call = createQuery(s"access_doc_role_${r.activity}", declaration(r), roles = docRole)
+
+ Then("a caller who may read every entity but lacks the doc's Role is refused by the doc's Role")
+ List(r.operator, r.certificate).foreach(grantRead(resourceUser2.userId, _))
+ val narrowed = makeGetRequest(call <@ (user2))
+ narrowed.code should equal(403)
+ messageOf(narrowed) should include(UserHasMissingRoles)
+
+ And("with the doc's Role as well, the query answers")
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser2.userId, docRole)
+ rows(makeGetRequest(call <@ (user2)))(everything) \ "operator_name" shouldBe JString("Acme Ltd")
+
+ And("a caller with only the doc's Role is refused for the entities it may not read")
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser3.userId, docRole)
+ messageOf(makeGetRequest(call <@ (user3))) shouldBe notReadable(r.operator, r.certificate)
+ }
+
+ scenario("a read-restricted field is null without its Role, copied with it, and cannot be filtered on without it") {
+ val r = registry(public = Set("activity", "operator", "country", "certificate"))
+ val call = createQuery(s"access_field_${r.activity}", declaration(r))
+ rows(makeGetRequest(call <@ (user2)))(everything) \ "operator_tax_number" shouldBe JNull
+
+ val filtered = createQuery(s"access_field_where_${r.activity}",
+ s"""{ "from": "${r.activity}", "join": [ { "entity": "${r.operator}", "on": "operator_id",
+ | "where": { "tax_number": "eq:DE-123" }, "fields": { "operator_name": "legal_name" } } ] }""".stripMargin)
+ val refused = makeGetRequest(filtered <@ (user2))
+ refused.code should equal(400)
+ messageOf(refused) should include(DynamicQueryInvalid)
+ messageOf(refused) should include("you may not read 'tax_number'")
+
+ When("the caller holds the field's read Role")
+ Entitlement.entitlement.vend.addEntitlement(DynamicEntitySpace.bankIdOrSystem(None), resourceUser2.userId,
+ s"CanGetDynamicEntityField_${r.operator}__tax_number")
+ rows(makeGetRequest(call <@ (user2)))(everything) \ "operator_tax_number" shouldBe JString("DE-123")
+ makeGetRequest(filtered <@ (user2)).code should equal(200)
+ }
+
+ scenario("a row-level entity joined in counts only the rows the caller's access list allows; personal records are never used") {
+ val r = registry(public = Set("activity", "operator", "country", "certificate"))
+ val Inspection = s"Inspection${r.activity}"
+ createDef(Inspection, s"""{"${idField(Inspection)}":{"type":"string"},"result":{"type":"string"},""" +
+ s""""activity_id":{"type":"reference:${r.activity}","indexed":true}}""", publicAccess = false, rowLevel = true)
+ val passed = saveRec(Inspection, "result" -> JString("passed"), "activity_id" -> JString(r.withEverything))
+ saveRec(Inspection, "result" -> JString("failed"), "activity_id" -> JString(r.withEverything))
+ And("an activity whose operator is someone's personal record")
+ val personalOperator = saveAs(r.operator, Some(resourceUser2.userId), "legal_name" -> JString("Personal Ltd"))
+ saveRec(r.activity, "name" -> JString("3 personal operator"), "operator_id" -> JString(personalOperator))
+
+ val call = createQuery(s"access_row_level_${r.activity}", declaration(r,
+ s""",
+ | { "entity": "$Inspection", "on": "activity_id", "cardinality": "many", "as": "inspections",
+ | "order": "earliest_by:result", "fields": { "result": "result" } }""".stripMargin))
+
+ Then("an anonymous caller sees no inspections: rows of a row-level entity need an access list entry")
+ rows(makeGetRequest(call))(everything) \ "inspections" shouldBe JArray(Nil)
+ When("user 2 may read one of the two inspections")
+ DynamicDataAccessProvider.provider.vend.grant(bankId = None, entityName = Inspection, dynamicDataId = passed,
+ userId = resourceUser2.userId, canRead = true, canUpdate = false, canDelete = false, canGrant = false, grantedBy = owner)
+ val forUser2 = rows(makeGetRequest(call <@ (user2)))
+ Then("user 2 sees that one only, and user 3 still sees none")
+ forUser2(everything) \ "inspections" \ "result" shouldBe JArray(List(JString("passed")))
+ rows(makeGetRequest(call <@ (user3)))(everything) \ "inspections" shouldBe JArray(Nil)
+ And("the personal operator is not copied, not even for the user who owns it")
+ forUser2("3 personal operator") \ "operator_name" shouldBe JNull
+ }
+ }
+
+ /** POST the declaration to Explain Dynamic Query; `sign` adds the caller's credentials to the request. */
+ private def explain(declarationText: String, sign: OBPReq => OBPReq, anonymous: Boolean = false, callerParameters: String = "") = {
+ val body = JObject(
+ JField("method_body", JString(URLEncoder.encode(declarationText, "UTF-8"))),
+ JField("caller_parameters", JString(callerParameters)),
+ JField("as_anonymous_caller", JBool(anonymous)))
+ makePostRequest(sign((baseRequest / "obp" / "v7.0.0" / "management" / "dynamic-resource-docs" / "explain").POST),
+ com.openbankproject.commons.util.JsonAliases.compactRender(body))
+ }
+
+ feature("Explain Dynamic Query: the reads a query would make, and the access it needs") {
+
+ scenario("the author sees every step, every entity with its Role, the restricted field, and the refusal a caller would get") {
+ val r = registry(public = Set("activity", "country"))
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString)
+
+ val forMe = explain(declaration(r), _ <@ (user1))
+ withClue(s"explain: ${forMe.body}") { forMe.code should equal(200) }
+ val json = forMe.body
+ (json \ "space") shouldBe JString("SYS")
+ (json \ "explained_for") shouldBe JString(s"user ${resourceUser1.userId}")
+ (json \ "caller_may_run") shouldBe JBool(false)
+ (json \ "refusal") shouldBe JString(notReadable(r.operator, r.certificate))
+
+ Then("every entity is listed in the query's order, with its Role, its public access and whether I may read it")
+ val entities = (json \ "entities").children
+ entities.map(e => (e \ "entity").values.toString) shouldBe List(r.activity, r.operator, r.country, r.certificate)
+ entities.map(e => (e \ "read_role").values.toString) shouldBe List(r.activity, r.operator, r.country, r.certificate).map("CanGetDynamicEntityRecord_" + _)
+ entities.map(e => (e \ "public_access").values) shouldBe List(true, false, true, false)
+ entities.map(e => (e \ "caller_may_read").values) shouldBe List(true, false, true, false)
+
+ And("the read-restricted field it copies is named with its Role")
+ ((json \ "restricted_fields").children.map(f => ((f \ "entity").values, (f \ "field").values, (f \ "read_role").values))) shouldBe
+ List((r.operator, "tax_number", s"CanGetDynamicEntityField_${r.operator}__tax_number"))
+
+ And("the steps are the page, then each join in order; the second certificate join reuses the first one's read")
+ val steps = (json \ "steps").children
+ steps.map(step => (step \ "step").values) shouldBe List(1, 2, 3, 4, 5)
+ val backends = steps.map(step => (step \ "backend").values.toString)
+ (backends(1), backends(2), backends(4)) shouldBe (("record provider", "record provider", "shared"))
+ (steps(1) \ "purpose").values.toString should include(s"follows '${r.activity}.operator_id' to '${r.operator}' (forward)")
+ (steps(3) \ "purpose").values.toString should include(s"'${r.certificate}' records whose 'activity_id' names the '${r.activity}' (reverse)")
+ (steps(4) \ "notes").children.head shouldBe JString("Uses the records already read for Join 3; nothing more is read.")
+ (json \ "rules").children should not be empty
+
+ When("it is explained for an anonymous caller")
+ val anonymous = explain(declaration(r), _ <@ (user1), anonymous = true).body
+ (anonymous \ "explained_for") shouldBe JString("an anonymous caller")
+ (anonymous \ "refusal") shouldBe JString(notReadable(r.operator, r.certificate))
+
+ When("I may read the two other entities")
+ List(r.operator, r.certificate).foreach(grantRead(resourceUser1.userId, _))
+ val allowed = explain(declaration(r), _ <@ (user1)).body
+ Then("the query can run for me, and nothing is refused")
+ (allowed \ "caller_may_run") shouldBe JBool(true)
+ (allowed \ "refusal") shouldBe JNothing
+ }
+
+ scenario("with the projection, the page and the reverse join show the SQL that would run, every value a ?") {
+ if (!IndexingCapabilities.projectionEnabled) cancel("needs the query projection (dynamic_entity.indexing.backend=auto)")
+ val r = registry(public = Set("activity", "operator", "country", "certificate"))
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString)
+ List(r.activity, r.certificate).foreach(entity => ProjectionProvisioner.ensureProvisioned(None, entity).unsafeRunSync())
+
+ val steps = (explain(declaration(r), _ <@ (user1), callerParameters = "obp_sort_by=name&obp_limit=10").body \ "steps").children
+ val page = steps.head
+ (page \ "backend") shouldBe JString("projection")
+ val pageSql = (page \ "sql").values.toString
+ pageSql should startWith("SELECT d.datajson FROM de_")
+ pageSql should include("d.bankid = ?")
+ pageSql should include("d.ispersonalentity = ?")
+ pageSql should include("ORDER BY")
+ pageSql should include("LIMIT ?")
+ (page \ "parameter_count").values shouldBe pageSql.count(_ == '?')
+ (steps(1) \ "purpose").values.toString should include("Count every match")
+
+ val reverse = steps.find(step => (step \ "purpose").values.toString.contains("(reverse)")).get
+ (reverse \ "backend") shouldBe JString("projection")
+ (reverse \ "sql").values.toString should include("IN (")
+ }
+
+ scenario("explaining needs the Role for writing Dynamic Resource Docs, and an invalid declaration is refused as Check would") {
+ val r = registry(public = Set("activity"))
+ val withoutRole = explain(declaration(r), _ <@ (user3))
+ withoutRole.code should equal(403)
+ messageOf(withoutRole) should include(UserHasMissingRoles)
+
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString)
+ val invalid = explain(s"""{ "from": "${r.activity}", "select": ["nope"] }""", _ <@ (user1))
+ invalid.code should equal(400)
+ messageOf(invalid) should include(s"${DynamicQueryInvalid}'select' names 'nope'")
+ }
+ }
+
+ private def dynamicEntityRequest: OBPReq = baseRequest / "obp" / "dynamic-entity"
+
+ feature("hide_field_from_public_access: a field of a public entity hidden from callers who reach it through public access") {
+
+ scenario("hidden in a Dynamic Query and on the public endpoint, shown with the entity's read Role, and never usable to filter by those who may not see it") {
+ val r = registry(public = Set("activity", "operator", "country", "certificate"))
+ val call = createQuery(s"access_public_hidden_${r.activity}",
+ s"""{ "from": "${r.activity}", "select": ["name"], "join": [ { "entity": "${r.operator}", "on": "operator_id",
+ | "fields": { "operator_name": "legal_name", "operator_contact": "contact_email" } } ] }""".stripMargin)
+ def contact(response: code.setup.APIResponse): JValue =
+ (response.body \ StringHelpers.snakify(r.activity).concat("_list")).children
+ .find(row => (row \ "name") == JString(everything)).map(_ \ "operator_contact").getOrElse(fail(s"no row in ${response.body}"))
+
+ Then("an anonymous caller, and a logged-in one without the operator's read Role, get null for it")
+ contact(makeGetRequest(call)) shouldBe JNull
+ contact(makeGetRequest(call <@ (user2))) shouldBe JNull
+ When("the caller holds the operator's read Role, their access is not public, and they see it")
+ grantRead(resourceUser2.userId, r.operator)
+ contact(makeGetRequest(call <@ (user2))) shouldBe JString("office@acme.example")
+
+ And("Explain names the field, the rule and the Role that lifts it")
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString)
+ val explained = explain(s"""{ "from": "${r.activity}", "join": [ { "entity": "${r.operator}", "on": "operator_id",
+ | "fields": { "operator_contact": "contact_email" } } ] }""".stripMargin, _ <@ (user1), anonymous = true).body
+ ((explained \ "restricted_fields").children.map(f => ((f \ "field").values, (f \ "restriction").values, (f \ "read_role").values, (f \ "caller_may_read").values))) shouldBe
+ List(("contact_email", "hide_field_from_public_access", s"CanGetDynamicEntityRecord_${r.operator}", false))
+
+ Then("the public endpoint leaves out both the hidden and the read-restricted field")
+ val publicList = makeGetRequest(dynamicEntityRequest / "public" / r.operator)
+ withClue(s"public: ${publicList.body}") { publicList.code should equal(200) }
+ val publicRecord = (publicList.body \ StringHelpers.snakify(r.operator).concat("_list")).children.head
+ (publicRecord \ "legal_name") shouldBe JString("Acme Ltd")
+ (publicRecord \ "contact_email") shouldBe JNothing
+ (publicRecord \ "tax_number") shouldBe JNothing
+
+ And("a public caller cannot filter by either, which would reveal their values")
+ val byContact = makeGetRequest(dynamicEntityRequest / "public" / r.operator < "office@acme.example"))
+ byContact.code should equal(400)
+ messageOf(byContact) shouldBe s"$DynamicEntityFieldNotReadable${r.operator}.contact_email."
+ messageOf(makeGetRequest(dynamicEntityRequest / "public" / r.operator < "DE-123"))) shouldBe
+ s"$DynamicEntityFieldNotReadable${r.operator}.tax_number."
+
+ And("on the authenticated endpoint, the read Role holder may filter by the hidden field but not by the one needing its own Role")
+ makeGetRequest(dynamicEntityRequest / r.operator <@ (user2) < "office@acme.example")).code should equal(200)
+ val byTax = makeGetRequest(dynamicEntityRequest / r.operator <@ (user2) < "DE-123"))
+ byTax.code should equal(400)
+ messageOf(byTax) shouldBe s"$DynamicEntityFieldNotReadable${r.operator}.tax_number."
+ }
+ }
+}
diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicQueryTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicQueryTest.scala
new file mode 100644
index 0000000000..57fce21c57
--- /dev/null
+++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicQueryTest.scala
@@ -0,0 +1,258 @@
+package code.api.v4_0_0
+
+import code.DynamicData.DynamicDataProvider
+import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON
+import cats.effect.unsafe.implicits.global
+import code.api.dynamic.entity.helper.DynamicEntitySpace
+import code.api.dynamic.entity.projection.{IndexingCapabilities, ProjectionProvisioner}
+import code.api.util.APIUtil.OAuth._
+import code.api.util.ApiRole
+import code.api.util.ErrorMessages.{DynamicCodeExecutionDisabled, DynamicQueryEntityNotReadable, DynamicQueryInvalid}
+import code.dynamicEntity.{DynamicEntityCommons, DynamicEntityProvider}
+import code.entitlement.Entitlement
+import com.openbankproject.commons.model.ErrorMessage
+import net.liftweb.util.StringHelpers
+import org.json4s._
+import org.json4s.native.Serialization.write
+
+import java.net.URLEncoder
+
+/**
+ * This suite checks Dynamic Queries end to end: a Dynamic Resource Doc with `programming_lang` `Query`
+ * is created over v4.0.0, checked against the entity definitions, served under the dynamic-endpoint
+ * prefix, and called: the entity read check, the joins, the envelope and count, and the caller's own
+ * list parameters. Also: a Dynamic Query must be GET, and needs no user-supplied code to be enabled;
+ * and the v7.0.0 dry-run compile checks one. The body parser on its own is covered by
+ * code.api.dynamic.entity.query.DynamicQueryDeclarationSpec, the joins by JoinSpec and
+ * DynamicEntityJoinPlanIntegrationTest.
+ */
+class DynamicQueryTest extends V400ServerSetup {
+
+ private val owner = "dynamic-query-owner"
+ private val sfx = java.util.UUID.randomUUID().toString.take(8).replace("-", "")
+ private val Operator = s"Operator$sfx"
+ private val Activity = s"Activity$sfx"
+ private val Certificate = s"Certificate$sfx"
+ private def idField(entity: String): String = StringHelpers.snakify(entity) + "_id"
+
+ private def createDef(entity: String, propsJson: String, rowLevel: Boolean = false, publicAccess: Boolean = false): Unit =
+ DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(
+ DynamicEntityCommons(entity, s"""{"$entity":{"properties":$propsJson}}""", None, owner, None, hasPersonalEntity = false,
+ hasPublicAccess = publicAccess, useRowLevelAccess = rowLevel)
+ ).openOrThrowException(s"failed to create definition for $entity")
+
+ private def saveRec(entity: String, fields: (String, JValue)*): String = {
+ val id = java.util.UUID.randomUUID().toString
+ val body = JObject(JField(idField(entity), JString(id)) :: fields.toList.map { case (k, v) => JField(k, v) })
+ DynamicDataProvider.connectorMethodProvider.vend.save(None, entity, body, Some(owner), false).openOrThrowException(s"failed to save $entity record")
+ id
+ }
+
+ private def grantRecordRead(entity: String): Unit =
+ Entitlement.entitlement.vend.addEntitlement(DynamicEntitySpace.bankIdOrSystem(None), resourceUser1.userId, s"CanGetDynamicEntityRecord_$entity")
+
+ // Braced body on purpose: .github/scripts/check_test_isolation.py only recognises `def name {` as a helper.
+ private def userCodeAllowed(allowed: Boolean): Unit = {
+ setPropsValues("allow_user_generated_scala_code" -> allowed.toString)
+ }
+
+ private def queryDoc(urlSegment: String, declaration: String, verb: String = "GET") =
+ SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy(
+ dynamicResourceDocId = None, bankId = None, roles = "",
+ partialFunctionName = s"dynamicQuery${urlSegment.capitalize}",
+ requestVerb = verb, requestUrl = s"/$urlSegment",
+ exampleRequestBody = None,
+ methodBody = URLEncoder.encode(declaration, "UTF-8"),
+ programmingLang = "Query")
+
+ private def create(doc: code.dynamicResourceDoc.JsonDynamicResourceDoc) =
+ makePostRequest((v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1), write(doc))
+
+ private def messageOf(response: code.setup.APIResponse): String = response.body.extract[ErrorMessage].message
+
+ private val activitiesQuery =
+ s"""{
+ | "from": "$Activity",
+ | "select": ["${idField(Activity)}", "name"],
+ | "join": [
+ | { "entity": "$Operator", "on": "operator_id", "fields": { "operator_legal_name": "legal_name" } },
+ | { "entity": "$Certificate", "on": "activity_id", "cardinality": "at_most_one", "pick": "latest_by:issue_date",
+ | "fields": { "latest_certificate": "number" } },
+ | { "entity": "$Certificate", "on": "activity_id", "cardinality": "exists", "as": "certified" }
+ | ],
+ | "envelope": { "rows": "activities", "count": "count" }
+ |}""".stripMargin
+
+ feature("Dynamic Query: a Dynamic Resource Doc whose body is a declaration (programming_lang Query)") {
+
+ scenario("create, check access, join, page and count") {
+ Given("operators, activities and certificates referring to the activities")
+ createDef(Operator, s"""{"${idField(Operator)}":{"type":"string"},"legal_name":{"type":"string"}}""")
+ createDef(Activity, s"""{"${idField(Activity)}":{"type":"string"},"name":{"type":"string","indexed":true},"operator_id":{"type":"reference:$Operator"}}""")
+ createDef(Certificate, s"""{"${idField(Certificate)}":{"type":"string"},"number":{"type":"string"},"issue_date":{"type":"DATE_WITH_DAY"},""" +
+ s""""activity_id":{"type":"reference:$Activity","indexed":true}}""")
+ val acme = saveRec(Operator, "legal_name" -> JString("Acme Ltd"))
+ val first = saveRec(Activity, "name" -> JString("a first"), "operator_id" -> JString(acme))
+ saveRec(Activity, "name" -> JString("b second"))
+ saveRec(Certificate, "number" -> JString("C-old"), "issue_date" -> JString("2026-01-01"), "activity_id" -> JString(first))
+ saveRec(Certificate, "number" -> JString("C-new"), "issue_date" -> JString("2026-06-01"), "activity_id" -> JString(first))
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString)
+
+ When("a Dynamic Query is created")
+ create(queryDoc(s"dq_$sfx/activities", activitiesQuery)).code should equal(201)
+ val call = dynamicEndpoint_Request / "dynamic-resource-doc" / s"dq_$sfx" / "activities"
+
+ Then("a caller who may not read the entities it reads is refused")
+ val refused = makeGetRequest(call.GET <@ (user1))
+ refused.code should equal(403)
+ def needs(entity: String) = s"$entity (needs CanGetDynamicEntityRecord_$entity at bank SYS)"
+ And("every entity it may not read is named in one answer, with the Role and bank it needs")
+ messageOf(refused) shouldBe s"$DynamicQueryEntityNotReadable${needs(Activity)}, ${needs(Operator)}, ${needs(Certificate)}."
+ grantRecordRead(Activity); grantRecordRead(Operator)
+ messageOf(makeGetRequest(call.GET <@ (user1))) shouldBe s"$DynamicQueryEntityNotReadable${needs(Certificate)}."
+ grantRecordRead(Certificate)
+
+ When("the caller may read all three")
+ val answered = makeGetRequest(call.GET <@ (user1) < "name"))
+ Then("the selected fields and the joins come back in the envelope, with the count")
+ answered.code should equal(200)
+ val rows = (answered.body \ "activities").asInstanceOf[JArray].arr
+ rows.map(_ \ "name") shouldBe List(JString("a first"), JString("b second"))
+ rows.head.asInstanceOf[JObject].obj.map(_._1) shouldBe List(idField(Activity), "name", "operator_legal_name", "latest_certificate", "certified")
+ rows.map(_ \ "operator_legal_name") shouldBe List(JString("Acme Ltd"), JNull)
+ rows.map(_ \ "latest_certificate") shouldBe List(JString("C-new"), JNull)
+ rows.map(_ \ "certified") shouldBe List(JBool(true), JBool(false))
+ (answered.body \ "count") shouldBe JInt(2)
+
+ When("the caller narrows it with the list endpoint's own parameters")
+ val paged = makeGetRequest(call.GET <@ (user1) < "name", "obp_sort_direction" -> "DESC", "obp_limit" -> "1"))
+ Then("one page comes back, and the count is still of every match")
+ ((paged.body \ "activities").asInstanceOf[JArray].arr.map(_ \ "name"), paged.body \ "count") shouldBe ((List(JString("b second")), JInt(2)))
+ val filtered = makeGetRequest(call.GET <@ (user1) < "eq:a first"))
+ (filtered.body \ "count") shouldBe JInt(1)
+
+ if (IndexingCapabilities.projectionEnabled) {
+ When("the projection is enabled and provisioned, so the page is read from it")
+ ProjectionProvisioner.ensureProvisioned(None, Activity).unsafeRunSync()
+ Then("the page and the count are the same")
+ makeGetRequest(call.GET <@ (user1) < "name")).body shouldBe answered.body
+ makeGetRequest(call.GET <@ (user1) < "name", "obp_sort_direction" -> "DESC", "obp_limit" -> "1")).body shouldBe paged.body
+
+ Given("a row-level inspection of the first activity that the caller may not read")
+ val Inspection = s"Inspection$sfx"
+ createDef(Inspection, s"""{"${idField(Inspection)}":{"type":"string"},"activity_id":{"type":"reference:$Activity","indexed":true}}""", rowLevel = true)
+ val inspection = saveRec(Inspection, "activity_id" -> JString(first))
+ ProjectionProvisioner.ensureProvisioned(None, Inspection).unsafeRunSync()
+ val withInspection = List("obp_exists[" + Inspection + "]" -> "via:activity_id")
+ Then("filtering by it finds nothing: rows the caller cannot read never count")
+ (makeGetRequest(call.GET <@ (user1) < s"eq:$acme"))
+ unindexed.code should equal(400)
+ messageOf(unindexed) should include(DynamicQueryInvalid)
+ }
+
+ scenario("a Dynamic Query is checked when created, must be GET, and needs no user-supplied code") {
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString)
+ val missing = create(queryDoc(s"dq_missing_$sfx", s"""{ "from": "NoSuchEntity$sfx" }"""))
+ missing.code should equal(400)
+ messageOf(missing) should include(s"${DynamicQueryInvalid}There is no Dynamic Entity 'NoSuchEntity$sfx' in this space.")
+
+ val malformed = create(queryDoc(s"dq_malformed_$sfx", s"""{ "from": "x", "joins": [] }"""))
+ malformed.code should equal(400)
+ messageOf(malformed) should include("unknown key 'joins'")
+
+ val posted = create(queryDoc(s"dq_post_$sfx", s"""{ "from": "x" }""", verb = "POST").copy(exampleRequestBody = Some(JObject())))
+ posted.code should equal(400)
+ messageOf(posted) should include("request_verb must be GET")
+
+ When("user-supplied code is switched off")
+ userCodeAllowed(false)
+ createDef(s"Plain$sfx", s"""{"${idField(s"Plain$sfx")}":{"type":"string"},"name":{"type":"string"}}""")
+ Then("a Dynamic Query can still be created, but a Scala body cannot")
+ create(queryDoc(s"dq_switched_off_$sfx", s"""{ "from": "Plain$sfx" }""")).code should equal(201)
+ val scala = create(queryDoc(s"dq_scala_$sfx", "").copy(methodBody = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.methodBody,
+ programmingLang = "Scala", requestVerb = "POST", exampleRequestBody = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.exampleRequestBody))
+ messageOf(scala) should include(DynamicCodeExecutionDisabled)
+ userCodeAllowed(true)
+ }
+
+ scenario("every call to a Dynamic Resource Doc is recorded as an API metric, refused or answered") {
+ setPropsValues("write_metrics" -> "true")
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString)
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.CanReadMetrics.toString)
+ val Audited = s"Audited$sfx"
+ createDef(Audited, s"""{"${idField(Audited)}":{"type":"string"},"name":{"type":"string"}}""")
+ saveRec(Audited, "name" -> JString("one"))
+ create(queryDoc(s"dq_audited_$sfx", s"""{ "from": "$Audited" }""")).code should equal(201)
+ val call = dynamicEndpoint_Request / "dynamic-resource-doc" / s"dq_audited_$sfx"
+
+ When("the query is called once without and once with read access to its entity")
+ makeGetRequest(call.GET <@ (user1)).code should equal(403)
+ grantRecordRead(Audited)
+ makeGetRequest(call.GET <@ (user1)).code should equal(200)
+ code.metrics.MetricBatchWriter.flush()
+
+ Then("both calls have a metric row, with their status, verb and the doc's function name")
+ val metrics = makeGetRequest((baseRequest / "obp" / "v6.0.0" / "management" / "metrics").GET <@ (user1) < s"/obp/dynamic-endpoint/dynamic-resource-doc/dq_audited_$sfx", "limit" -> "10"))
+ metrics.code should equal(200)
+ val rows = (metrics.body \ "metrics").children
+ rows.map(row => (row \ "status_code").extract[Int]).sorted shouldBe List(200, 403)
+ rows.foreach { row =>
+ (row \ "verb").extract[String] shouldBe "GET"
+ (row \ "user_id").extract[String] shouldBe resourceUser1.userId
+ (row \ "implemented_by_partial_function").extract[String] should startWith(s"dynamicQueryDq_audited_$sfx")
+ }
+ }
+
+ scenario("a Dynamic Query is public when its doc has no Roles and every entity it reads has public access") {
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString)
+ val Published = s"Published$sfx"; val Internal = s"Internal$sfx"
+ createDef(Published, s"""{"${idField(Published)}":{"type":"string"},"name":{"type":"string"},"internal_id":{"type":"reference:$Internal"}}""",
+ publicAccess = true)
+ createDef(Internal, s"""{"${idField(Internal)}":{"type":"string"},"note":{"type":"string"}}""")
+ val internal = saveRec(Internal, "note" -> JString("staff only"))
+ saveRec(Published, "name" -> JString("open data"), "internal_id" -> JString(internal))
+ def publicDoc(segment: String, declaration: String) =
+ queryDoc(segment, declaration).copy(errorResponseBodies = "OBP-50000: Unknown Error.")
+
+ When("a query reads only the entity with public access, and its doc has no Roles")
+ create(publicDoc(s"dq_public_$sfx", s"""{ "from": "$Published", "select": ["name"] }""")).code should equal(201)
+ Then("anyone can call it, without logging in")
+ val anonymous = makeGetRequest(dynamicEndpoint_Request / "dynamic-resource-doc" / s"dq_public_$sfx")
+ anonymous.code should equal(200)
+ (anonymous.body \ StringHelpers.snakify(Published).concat("_list") \ "name") shouldBe JArray(List(JString("open data")))
+
+ When("a query also joins an entity without public access")
+ create(publicDoc(s"dq_public_join_$sfx",
+ s"""{ "from": "$Published", "join": [ { "entity": "$Internal", "on": "internal_id", "fields": { "note": "note" } } ] }""")).code should equal(201)
+ Then("an anonymous caller is refused, and told which entity and Role it would need")
+ val refused = makeGetRequest(dynamicEndpoint_Request / "dynamic-resource-doc" / s"dq_public_join_$sfx")
+ refused.code should equal(403)
+ messageOf(refused) shouldBe s"$DynamicQueryEntityNotReadable$Internal (needs CanGetDynamicEntityRecord_$Internal at bank SYS)."
+ }
+
+ scenario("the v7.0.0 dry-run compile checks a Dynamic Query") {
+ Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString)
+ createDef(s"Checked$sfx", s"""{"${idField(s"Checked$sfx")}":{"type":"string"},"name":{"type":"string"}}""")
+ def compile(declaration: String) = makePostRequest(
+ (baseRequest / "obp" / "v7.0.0" / "management" / "dynamic-resource-docs" / "compile").POST <@ (user1),
+ write(Map("request_verb" -> "GET", "request_url" -> "/checked", "programming_lang" -> "Query",
+ "method_body" -> URLEncoder.encode(declaration, "UTF-8"))))
+ val good = compile(s"""{ "from": "Checked$sfx", "select": ["name"] }""")
+ good.code should equal(200)
+ (good.body \ "compiles") shouldBe JBool(true)
+ val bad = compile(s"""{ "from": "Checked$sfx", "select": ["nope"] }""")
+ (bad.body \ "compiles") shouldBe JBool(false)
+ ((bad.body \ "errors")(0) \ "message").values.toString should include("'select' names 'nope'")
+ }
+ }
+}
diff --git a/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityJoinPlanIntegrationTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityJoinPlanIntegrationTest.scala
new file mode 100644
index 0000000000..7390b3589e
--- /dev/null
+++ b/obp-api/src/test/scala/code/api/v6_0_0/DynamicEntityJoinPlanIntegrationTest.scala
@@ -0,0 +1,232 @@
+/**
+Open Bank Project - API
+Copyright (C) 2011-2026, TESOBE GmbH.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+ */
+
+package code.api.v6_0_0
+
+import code.DynamicData.{DynamicDataAccessProvider, DynamicDataProvider}
+import code.api.dynamic.entity.helper.DynamicEntitySpace
+import code.api.dynamic.entity.projection.{IndexingCapabilities, ProjectionProvisioner}
+import code.api.dynamic.entity.query._
+import code.dynamicEntity.{DynamicEntityCommons, DynamicEntityProvider}
+import code.entitlement.Entitlement
+import cats.effect.unsafe.implicits.global
+import net.liftweb.util.StringHelpers
+import org.json4s.JsonAST._
+
+/**
+ * This suite checks the join machinery of Dynamic Queries against the real record store: joins planned
+ * from stored definitions, in both directions (forward: the record's reference field names the other
+ * record; reverse: the other records' reference field names this one), applied with the access rules:
+ * shared records only, the row-level access list, and read-restricted fields.
+ *
+ * Neither direction needs the PostgreSQL projection, so this runs on any database. When the projection
+ * is enabled, the reverse scenario also provisions it and checks that the indexed lookup gives the same
+ * result as the in-memory one. Entity names carry a per-run suffix so reruns never see stale data. The
+ * planner's checks and the merge on their own are covered by code.api.dynamic.entity.query.JoinSpec.
+ */
+class DynamicEntityJoinPlanIntegrationTest extends V600ServerSetup {
+
+ private val owner = "join-plan-owner"
+ private val userA = "join-plan-user-a"
+ private val userB = "join-plan-user-b"
+ private def suffix(): String = java.util.UUID.randomUUID().toString.take(8).replace("-", "")
+ private def idField(entity: String): String = StringHelpers.snakify(entity) + "_id"
+
+ private def createDef(entity: String, propsJson: String, rowLevel: Boolean = false): Unit =
+ DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(
+ DynamicEntityCommons(entity, s"""{"$entity":{"properties":$propsJson}}""", None, owner, None,
+ hasPersonalEntity = true, hasCommunityAccess = true, useRowLevelAccess = rowLevel)
+ ).openOrThrowException(s"failed to create definition for $entity")
+
+ /** Save a record with an explicit id, shared unless `personalOwner` is given. Returns the id. */
+ private def saveRec(entity: String, personalOwner: Option[String], fields: (String, JValue)*): String = {
+ val id = java.util.UUID.randomUUID().toString
+ val body = JObject(JField(idField(entity), JString(id)) :: fields.toList.map { case (k, v) => JField(k, v) })
+ DynamicDataProvider.connectorMethodProvider.vend.save(None, entity, body, personalOwner.orElse(Some(owner)), personalOwner.isDefined)
+ .openOrThrowException(s"failed to save $entity record")
+ id
+ }
+
+ private def grantRead(entity: String, recordId: String, userId: String): Unit =
+ DynamicDataAccessProvider.provider.vend.grant(bankId = None, entityName = entity, dynamicDataId = recordId,
+ userId = userId, canRead = true, canUpdate = false, canDelete = false, canGrant = false, grantedBy = owner)
+
+ /** The parent entity's shared records, ordered by `name` so assertions can be positional. */
+ private def page(entity: String): List[JObject] =
+ DynamicDataProvider.connectorMethodProvider.vend.getAllDataJson(None, entity, None, isPersonalEntity = false)
+ .sortBy(r => (r \ "name").values.toString)
+
+ private def column(records: List[JObject], field: String): List[JValue] = records.map(_ \ field)
+
+ private def plan(parent: String, joins: List[JoinRequest], caller: Option[String]): JoinPlan =
+ JoinPlanner.planFor(None, parent, joins, caller, "", _ => true).fold(e => fail(e.message), identity)
+
+ feature("Forward joins: the record's reference field names the other record") {
+ scenario("shared, personal, dangling and missing references, a row-level target, and a read-restricted field") {
+ val sfx = suffix()
+ val Operator = s"Operator$sfx"; val Licence = s"Licence$sfx"; val Activity = s"Activity$sfx"
+ Given("operators (one field read-restricted), row-level licences, and activities referencing both")
+ createDef(Operator, s"""{"${idField(Operator)}":{"type":"string"},"legal_name":{"type":"string"},"tax_number":{"type":"string","read_role_required":true}}""")
+ createDef(Licence, s"""{"${idField(Licence)}":{"type":"string"},"number":{"type":"string"}}""", rowLevel = true)
+ createDef(Activity, s"""{"${idField(Activity)}":{"type":"string"},"name":{"type":"string"},"city":{"type":"string"},""" +
+ s""""operator_id":{"type":"reference:$Operator"},"licence_id":{"type":"reference:$Licence"}}""")
+
+ val acme = saveRec(Operator, None, "legal_name" -> JString("Acme Ltd"), "tax_number" -> JString("DE-123"))
+ val privateOperator = saveRec(Operator, Some(userB), "legal_name" -> JString("Someone's own operator"))
+ val licenceForA = saveRec(Licence, None, "number" -> JString("L-1"))
+ grantRead(Licence, licenceForA, userA)
+
+ saveRec(Activity, None, "name" -> JString("1 acme"), "operator_id" -> JString(acme), "licence_id" -> JString(licenceForA))
+ saveRec(Activity, None, "name" -> JString("2 private operator"), "operator_id" -> JString(privateOperator))
+ saveRec(Activity, None, "name" -> JString("3 dangling"), "operator_id" -> JString("no-such-operator"))
+ saveRec(Activity, None, "name" -> JString("4 no reference"))
+
+ val joins = List(
+ JoinRequest(Operator, "operator_id", fields = List("operator_legal_name" -> "legal_name", "operator_tax_number" -> "tax_number")),
+ JoinRequest(Licence, "licence_id", fields = List("licence_number" -> "number")))
+
+ Then("a planning mistake is reported against the stored definition")
+ JoinPlanner.planFor(None, Activity, List(JoinRequest(Operator, "city", fields = List("x" -> "legal_name"))), Some(userA), "", _ => true)
+ .left.map(_.message) shouldBe Left(s"Join 1 ('$Operator' on 'city'): 'city' must be a reference field linking '$Activity' and '$Operator', " +
+ s"on either of them, but '$Activity' has no field 'city' typed 'reference:$Operator' and '$Operator' has no field 'city' typed 'reference:$Activity'.")
+
+ When("user A, who may read the licence but lacks the tax number's read role, expands the page")
+ val forA = plan(Activity, joins, Some(userA))(page(Activity), None, Some(userA), "")
+ Then("shared operators are copied; another user's personal operator and a dangling reference give null")
+ column(forA, "operator_legal_name") shouldBe List(JString("Acme Ltd"), JNull, JNull, JNull)
+ And("the read-restricted tax number is null without its role")
+ column(forA, "operator_tax_number") shouldBe List(JNull, JNull, JNull, JNull)
+ And("the row-level licence is copied, because A's access list allows it")
+ column(forA, "licence_number").head shouldBe JString("L-1")
+
+ When("user B expands the same page")
+ val forB = plan(Activity, joins, Some(userB))(page(Activity), None, Some(userB), "")
+ Then("the licence is null for B, whose access list does not include it")
+ column(forB, "licence_number").head shouldBe JNull
+ And("B's own personal operator is still not copied: only shared records are used")
+ column(forB, "operator_legal_name")(1) shouldBe JNull
+
+ When("user A is granted the tax number's read role")
+ Entitlement.entitlement.vend.addEntitlement(DynamicEntitySpace.bankIdOrSystem(None), userA, s"CanGetDynamicEntityField_${Operator}__tax_number")
+ Then("the tax number is copied for A")
+ column(plan(Activity, joins, Some(userA))(page(Activity), None, Some(userA), ""), "operator_tax_number").head shouldBe JString("DE-123")
+
+ And("the page keeps its records and order")
+ forA.map(r => (r \ "name").values.toString) shouldBe List("1 acme", "2 private operator", "3 dangling", "4 no reference")
+ }
+ }
+
+ feature("Reverse joins: the other records' reference field names this record") {
+ scenario("at_most_one with pick, many with order, exists with where and custom values, and a row-level related entity") {
+ val sfx = suffix()
+ val Activity = s"Activity$sfx"; val Certificate = s"Certificate$sfx"; val Inspection = s"Inspection$sfx"
+ Given("activities, certificates referring to them (indexed), and row-level inspections referring to them")
+ createDef(Activity, s"""{"${idField(Activity)}":{"type":"string"},"name":{"type":"string"}}""")
+ createDef(Certificate, s"""{"${idField(Certificate)}":{"type":"string"},"number":{"type":"string"},"status":{"type":"string"},""" +
+ s""""issue_date":{"type":"DATE_WITH_DAY"},"activity_id":{"type":"reference:$Activity","indexed":true},""" +
+ s""""unindexed_activity_id":{"type":"reference:$Activity"}}""")
+ createDef(Inspection, s"""{"${idField(Inspection)}":{"type":"string"},"activity_id":{"type":"reference:$Activity","indexed":true}}""", rowLevel = true)
+
+ val withCertificates = saveRec(Activity, None, "name" -> JString("1 with certificates"))
+ val withNone = saveRec(Activity, None, "name" -> JString("2 with none"))
+ saveRec(Certificate, None, "number" -> JString("C-1"), "status" -> JString("valid"), "issue_date" -> JString("2026-01-01"), "activity_id" -> JString(withCertificates))
+ saveRec(Certificate, None, "number" -> JString("C-2"), "status" -> JString("revoked"), "issue_date" -> JString("2026-06-01"), "activity_id" -> JString(withCertificates))
+ saveRec(Certificate, None, "number" -> JString("C-3"), "status" -> JString("valid"), "activity_id" -> JString(withCertificates))
+ saveRec(Certificate, Some(userB), "number" -> JString("C-personal"), "status" -> JString("valid"), "issue_date" -> JString("2027-01-01"), "activity_id" -> JString(withCertificates))
+ grantRead(Inspection, saveRec(Inspection, None, "activity_id" -> JString(withCertificates)), userA)
+ saveRec(Inspection, None, "activity_id" -> JString(withNone))
+
+ val joins = List(
+ JoinRequest(Certificate, "activity_id", cardinality = Some("at_most_one"), pick = Some("latest_by:issue_date"),
+ fields = List("latest_certificate_number" -> "number", "latest_certificate_issue_date" -> "issue_date")),
+ JoinRequest(Certificate, "activity_id", cardinality = Some("at_most_one"), pick = Some("latest_by:issue_date"),
+ where = List(Filter("status", FilterOp.Eq, List("valid"))), fields = List("latest_valid_certificate_number" -> "number")),
+ JoinRequest(Certificate, "activity_id", cardinality = Some("many"), as = Some("certificates"), order = Some("earliest_by:issue_date"),
+ fields = List("number" -> "number", "status" -> "status")),
+ JoinRequest(Certificate, "activity_id", cardinality = Some("exists"), as = Some("revocation"),
+ where = List(Filter("status", FilterOp.Eq, List("revoked"))),
+ trueValue = Some(JString("revoked")), falseValue = Some(JString("clean"))),
+ JoinRequest(Inspection, "activity_id", cardinality = Some("exists"), as = Some("inspected")))
+
+ Then("a reference field that is not indexed is rejected with the field to fix")
+ JoinPlanner.planFor(None, Activity, List(JoinRequest(Certificate, "unindexed_activity_id", cardinality = Some("exists"), as = Some("x"))), Some(userA), "", _ => true)
+ .left.map(_.message).left.getOrElse("") should include("'unindexed_activity_id' on '" + Certificate + "' must be declared \"indexed\": true")
+
+ When("user A applies them to the page")
+ val forA = plan(Activity, joins, Some(userA))(page(Activity), None, Some(userA), "")
+ Then("at_most_one picks the latest dated certificate; the undated one and another user's personal one are never chosen")
+ column(forA, "latest_certificate_number") shouldBe List(JString("C-2"), JNull)
+ column(forA, "latest_certificate_issue_date") shouldBe List(JString("2026-06-01"), JNull)
+ And("the where filter is applied before the pick")
+ column(forA, "latest_valid_certificate_number") shouldBe List(JString("C-1"), JNull)
+ And("many lists the shared certificates, earliest first, the undated one last")
+ (forA.head \ "certificates" \ "number") shouldBe JArray(List(JString("C-1"), JString("C-2"), JString("C-3")))
+ (forA(1) \ "certificates") shouldBe JArray(Nil)
+ And("exists gives the custom values")
+ column(forA, "revocation") shouldBe List(JString("revoked"), JString("clean"))
+ And("the row-level inspection counts only where A's access list allows it")
+ column(forA, "inspected") shouldBe List(JBool(true), JBool(false))
+
+ When("user B applies the same")
+ val forB = plan(Activity, joins, Some(userB))(page(Activity), None, Some(userB), "")
+ Then("B sees no inspection, and B's own personal certificate is still not used")
+ column(forB, "inspected") shouldBe List(JBool(false), JBool(false))
+ column(forB, "latest_certificate_number").head shouldBe JString("C-2")
+
+ if (IndexingCapabilities.projectionEnabled) {
+ When("the projection is enabled, provision it so the lookup goes through the indexed column")
+ List(Certificate, Inspection).foreach(e => ProjectionProvisioner.ensureProvisioned(None, e).unsafeRunSync())
+ Then("the indexed lookup gives exactly the same result")
+ plan(Activity, joins, Some(userA))(page(Activity), None, Some(userA), "") shouldBe forA
+ }
+ }
+ }
+
+ feature("A self-reference needs a direction") {
+ scenario("an employee's manager (forward) and direct reports (reverse) through the same field") {
+ val Employee = s"Employee${suffix()}"
+ createDef(Employee, s"""{"${idField(Employee)}":{"type":"string"},"name":{"type":"string"},"manager_id":{"type":"reference:$Employee","indexed":true}}""")
+ val boss = saveRec(Employee, None, "name" -> JString("1 boss"))
+ saveRec(Employee, None, "name" -> JString("2 alice"), "manager_id" -> JString(boss))
+ saveRec(Employee, None, "name" -> JString("3 bob"), "manager_id" -> JString(boss))
+
+ Then("without a direction the join is refused, with both readings explained")
+ JoinPlanner.planFor(None, Employee, List(JoinRequest(Employee, "manager_id", fields = List("manager_name" -> "name"))), Some(userA), "", _ => true)
+ .left.map(_.message).left.getOrElse("") should include("Say which with \"direction\"")
+
+ When("both directions are given")
+ val joined = plan(Employee, List(
+ JoinRequest(Employee, "manager_id", direction = Some("forward"), fields = List("manager_name" -> "name")),
+ JoinRequest(Employee, "manager_id", direction = Some("reverse"), cardinality = Some("many"), as = Some("reports"),
+ order = Some("earliest_by:name"), fields = List("name" -> "name"))), Some(userA))(page(Employee), None, Some(userA), "")
+ Then("each employee gets their manager, and the boss gets their reports")
+ column(joined, "manager_name") shouldBe List(JNull, JString("1 boss"), JString("1 boss"))
+ (joined.head \ "reports" \ "name") shouldBe JArray(List(JString("2 alice"), JString("3 bob")))
+ (joined(1) \ "reports") shouldBe JArray(Nil)
+ }
+ }
+}
diff --git a/obp-api/src/test/scala/code/api/v7_0_0/DynamicEntityDeleteEntitlementsTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/DynamicEntityDeleteEntitlementsTest.scala
new file mode 100644
index 0000000000..060558e26f
--- /dev/null
+++ b/obp-api/src/test/scala/code/api/v7_0_0/DynamicEntityDeleteEntitlementsTest.scala
@@ -0,0 +1,125 @@
+/**
+Open Bank Project - API
+Copyright (C) 2011-2026, TESOBE GmbH.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+ */
+
+package code.api.v7_0_0
+
+import code.api.Constant.DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID
+import code.api.util.APIUtil.OAuth._
+import code.api.util.ApiRole
+import code.api.util.ApiRole._
+import code.entitlement.Entitlement
+import code.setup.ServerSetupWithTestData
+import com.github.dwickern.macros.NameOf.nameOf
+import com.openbankproject.commons.util.ApiVersion
+import org.json4s.JsonDSL._
+import org.json4s._
+import org.json4s.native.JsonMethods.parse
+import org.json4s.native.Serialization.write
+import org.scalatest.Tag
+
+import java.util.UUID
+
+/**
+ * Deleting a Dynamic Entity deletes the grants of its record Roles, but only in its own space.
+ *
+ * A record Role is named after the entity alone (CanGetDynamicEntityRecord_), with no bank, so
+ * an entity of the same name at another bank, or at SYS, uses the same Role names. Deleting one of
+ * them used to delete every grant of those names at every bank, and to unregister the Roles while the
+ * other entity still used them.
+ */
+class DynamicEntityDeleteEntitlementsTest extends ServerSetupWithTestData {
+
+ object VersionOfApi extends Tag(ApiVersion.v7_0_0.toString)
+ object ApiEndpoint1 extends Tag(nameOf(Http4s700DynamicEntityDefinitions.deleteDynamicEntityDefinitionCascade))
+
+ private val SYS = DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID
+
+ def v7 = baseRequest / "obp" / "v7.0.0"
+
+ private def definitionsAt(bankId: String) = v7 / "management" / "banks" / bankId / "dynamic-entities"
+
+ private def newEntityName(): String = "test_delete_grants_" + UUID.randomUUID().toString.take(8).replace("-", "")
+
+ private def definition(entityName: String): JValue =
+ ("entity_name" -> entityName) ~
+ ("has_personal_entity" -> false) ~
+ ("schema" -> parse(
+ """{"description": "Entity for the delete-grants test.", "required": ["name"],
+ | "properties": {"name": {"type": "string", "maxLength": 40, "minLength": 1, "example": "Test"}}}""".stripMargin))
+
+ private def grant(bankId: String, userId: String, roleName: String): Unit =
+ Entitlement.entitlement.vend.addEntitlement(bankId, userId, roleName)
+
+ private def createdAt(bankId: String, entityName: String): String = {
+ grant(bankId, resourceUser1.userId, canCreateDynamicEntityDefinition.toString)
+ val response = makePostRequest(definitionsAt(bankId).POST <@ (user1), write(definition(entityName)))
+ response.code should equal(201)
+ (response.body \ "dynamic_entity_id").extract[String]
+ }
+
+ private def cascadeDelete(bankId: String, dynamicEntityId: String): Unit = {
+ grant(bankId, resourceUser1.userId, canDeleteCascadeDynamicEntityDefinition.toString)
+ makeDeleteRequest((definitionsAt(bankId) / "cascade" / dynamicEntityId).DELETE <@ (user1)).code should equal(204)
+ }
+
+ /** The bank ids at which user2 holds `roleName`. */
+ private def user2BanksFor(roleName: String): Set[String] =
+ Entitlement.entitlement.vend.getEntitlementsByUserId(resourceUser2.userId).openOr(Nil)
+ .filter(_.roleName == roleName).map(_.bankId).toSet
+
+ feature("Deleting a Dynamic Entity leaves the grants of another space's entity of the same name") {
+
+ scenario("an entity named the same at a bank and at SYS: deleting one leaves the other's grants", ApiEndpoint1, VersionOfApi) {
+ val entityName = newEntityName()
+ val readRole = s"CanGetDynamicEntityRecord_$entityName"
+ val bank = testBankId1.value
+
+ Given(s"$entityName at ${bank} and at SYS, and user2 granted its read Role at both, and at the empty bank id")
+ val atBank = createdAt(bank, entityName)
+ val atSys = createdAt(SYS, entityName)
+ grant(bank, resourceUser2.userId, readRole)
+ grant(SYS, resourceUser2.userId, readRole)
+ grant("", resourceUser2.userId, readRole)
+ user2BanksFor(readRole) should equal(Set(bank, SYS, ""))
+
+ When("the system level entity is deleted")
+ cascadeDelete(SYS, atSys)
+
+ Then("only the system space's grants are gone, SYS and the empty bank id")
+ user2BanksFor(readRole) should equal(Set(bank))
+ And("the Role is still registered, because the bank's entity still uses it")
+ ApiRole.availableRoles should contain(readRole)
+
+ When("the bank's entity is deleted too")
+ cascadeDelete(bank, atBank)
+
+ Then("its grant goes, and with no entity of that name left the Role is no longer registered")
+ user2BanksFor(readRole) shouldBe empty
+ ApiRole.availableRoles should not contain readRole
+ }
+ }
+}
diff --git a/scripts/resource_doc_baseline/parity_allowlist.json b/scripts/resource_doc_baseline/parity_allowlist.json
index 6f48df4ba2..da3cec4f15 100644
--- a/scripts/resource_doc_baseline/parity_allowlist.json
+++ b/scripts/resource_doc_baseline/parity_allowlist.json
@@ -1183,41 +1183,41 @@
"version": "v6_0_0",
"endpoint": "createBankLevelDynamicEntity",
"field": "description",
- "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed.",
+ "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed. Digest refresh 2026-10-02: documents the hide_field_from_public_access field flag.",
"lift_digest": "2e7c649041362776f7eec66ccbb9f025f3edfead992108d0fd5864af76e93961",
- "http4s_digest": "2177cf2f3d19eed21d73bacb62cf340e195298eb334857a67e24b4ccdaab61ce"
+ "http4s_digest": "6bf7d3278c575dfe04a64e3f20fe7bafbdfccfdc5aff61dafefd73563a20de56"
},
{
"version": "v6_0_0",
"endpoint": "createSystemDynamicEntity",
"field": "description",
- "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed.",
+ "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed. Digest refresh 2026-10-02: documents the hide_field_from_public_access field flag.",
"lift_digest": "81310cdd2be1c2b33d465cabd1604cedee38bfa1ecccae4b016d0f99aa443ab6",
- "http4s_digest": "a216f7033fb45112130ff7ae7acee56b02ee0443d428e60fcaee1415ba4083b4"
+ "http4s_digest": "afe200b33fc38afdb0a212a229e6cec6ccf7caa77b2ac7ff23633fd16638432b"
},
{
"version": "v6_0_0",
"endpoint": "updateBankLevelDynamicEntity",
"field": "description",
- "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed.",
+ "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed. Digest refresh 2026-10-02: documents the hide_field_from_public_access field flag.",
"lift_digest": "2d18665f9db52506a1b043ad7575b16ac52cedf0d11f036afd4a96b353eb8ddd",
- "http4s_digest": "0853c5554b070120aa477631bb4ce97f36b60e09acaad61f64199d407fd3d1da"
+ "http4s_digest": "67aea2489cfbaf7b62ab7fe4c1a7fa7c5ad904bb4bd5db5a0977eeb37298a1cb"
},
{
"version": "v6_0_0",
"endpoint": "updateMyDynamicEntity",
"field": "description",
- "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed.",
+ "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed. Digest refresh 2026-10-02: documents the hide_field_from_public_access field flag.",
"lift_digest": "97f0c97cc592f89f4dba8a6641ad4b41f6ffa6f4f6e78bf3d3cc34a6a11f76e7",
- "http4s_digest": "c8b602bc69dc86b8eb92e52f18e0a2714ef0d14d5de560186da8a372c6af2ecf"
+ "http4s_digest": "b3d573ff89040298846a54f3ce897f96f6abac1de477c9ea4efe267e8c8f13ff"
},
{
"version": "v6_0_0",
"endpoint": "updateSystemDynamicEntity",
"field": "description",
- "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed.",
+ "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed. Digest refresh 2026-10-02: documents the hide_field_from_public_access field flag.",
"lift_digest": "e28c2996f3fd54842534fd54d45d7ba49df8b346e65f541353be4fb5311b550c",
- "http4s_digest": "787e40986d922b9f0d28ce3f1b3f5110f8521bc2a26fab8b3923e6851c41bfc4"
+ "http4s_digest": "6f71dd7c0ad0fcbdc973f5bebbed11b75c57a7f6bef56613d92a999b12bf6821"
},
{
"version": "v3_0_0",