Dynamic Entity joins and updates

.gitignore

@@ -8,7 +8,7 @@
 .settings
 .metals
 .vscode
-.claude/settings.local.json
+.claude/
 *.code-workspace
 .zed
 .cursor

obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala

@@ -27,7 +27,7 @@ TESOBE (http://www.tesobe.com/)
 
 package code.api.ResourceDocs1_4_0
 
-import code.api.Constant.{GET_DYNAMIC_RESOURCE_DOCS_TTL, GET_STATIC_RESOURCE_DOCS_TTL, HostName, PARAM_LOCALE}
+import code.api.Constant.{DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID, GET_DYNAMIC_RESOURCE_DOCS_TTL, GET_STATIC_RESOURCE_DOCS_TTL, HostName, PARAM_LOCALE}
 import code.api.OBPRestHelper
 import code.api.cache.Caching
 import code.api.util.APIUtil._
@@ -297,6 +297,54 @@ trait ResourceDocsAPIMethods extends MdcLoggable {
       List(apiTagDocumentation, apiTagApi)
     )
 
+    localResourceDocs += ResourceDoc(
+      implementedInApiVersion,
+      "getBankLevelDynamicResourceDocsOpenAPI31",
+      "GET",
+      "/banks/BANK_ID/resource-docs/API_VERSION/openapi",
+      "Get Bank Level Dynamic OpenAPI 3.1 documentation",
+      s"""Returns OpenAPI 3.1 documentation for the dynamic resources of one bank: its Dynamic Entities,
+         |Dynamic Endpoints and Dynamic Resource Docs.
+         |
+         |BANK_ID is the bank whose dynamic resources you want. Use ${DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID} for the
+         |system level dynamic resources, the ones that belong to no bank.
+         |
+         |API_VERSION is the version you want documentation about e.g. v7.0.0
+         |
+         |Static endpoints belong to no bank, so this document only ever contains dynamic resources. To
+         |document every bank's dynamic resources at once, use /resource-docs/API_VERSION/openapi?content=dynamic
+         |
+         |## Query Parameters
+         |
+         |**tags** - Filter by endpoint tags (comma-separated list). Empty values will return error OBP-10053
+         |
+         |**functions** - Filter by function names (comma-separated list). Empty values will return error OBP-10054
+         |
+         |**locale** - Language for localized documentation, e.g. ?locale=en_GB. Invalid locales will return error OBP-10041
+         |
+         |For YAML format, use the corresponding endpoint: /banks/BANK_ID/resource-docs/API_VERSION/openapi.yaml
+         |
+         |Note: Resource Docs are cached, TTL is ${GET_DYNAMIC_RESOURCE_DOCS_TTL} seconds
+         |
+         |## Examples
+         |
+         |${getObpApiRoot}/v7.0.0/banks/${DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID}/resource-docs/v7.0.0/openapi
+         |${getObpApiRoot}/v7.0.0/banks/BANK_ID/resource-docs/v7.0.0/openapi.yaml
+         |
+      """,
+      EmptyBody,
+      EmptyBody,
+      BankNotFound ::
+      InvalidApiVersionString ::
+      ApiVersionNotSupported ::
+      InvalidLocale ::
+      InvalidTagsParameter ::
+      InvalidFunctionsParameter ::
+      UnknownError :: Nil,
+      List(apiTagDocumentation, apiTagApi),
+      Some(List(canReadDynamicResourceDocsAtOneBank))
+    )
+
     implicit val formats = CustomJsonFormats.rolesMappedToClassesFormats
 
     // avoid repeat execute method getSpecialInstructions, here save the calculate results.
@@ -507,7 +555,7 @@ trait ResourceDocsAPIMethods extends MdcLoggable {
       // (don't keep a stale value left over from a different aggregated request that may have
       // overwritten this var on the shared ResourceDoc).
       val dynamicDocs = allDynamicResourceDocs
-        .filter(rd => if (bankId.isDefined) rd.createdByBankId == bankId else true)
+        .filter(rd => bankId.forall(space => APIUtil.dynamicResourceDocBelongsToSpace(rd, space)))
         .map { it =>
           it.specifiedUrl = if (it.partialFunctionName.startsWith("dynamicEntity")) Some(s"/${it.implementedInApiVersion.urlPrefix}/${ApiVersion.`dynamic-entity`}${it.requestUrl}") else Some(s"/${it.implementedInApiVersion.urlPrefix}/${ApiVersion.`dynamic-endpoint`}${it.requestUrl}")
           it
@@ -1147,36 +1195,36 @@ trait ResourceDocsAPIMethods extends MdcLoggable {
 
 
 
-    def convertResourceDocsToOpenAPI31YAMLAndSetCache(cacheKey: String, requestedApiVersionString: String, resourceDocsJson: List[JSONFactory1_4_0.ResourceDocJson]) : String = {
+    def convertResourceDocsToOpenAPI31YAMLAndSetCache(cacheKey: String, requestedApiVersionString: String, resourceDocsJson: List[JSONFactory1_4_0.ResourceDocJson], setCache: (String, String) => Unit = Caching.setStaticSwaggerDocCache) : String = {
       logger.debug(s"Generating OpenAPI 3.1 YAML-convertResourceDocsToOpenAPI31YAMLAndSetCache requestedApiVersion is $requestedApiVersionString")
       val hostname = HostName
       val openApiDoc = code.api.ResourceDocs1_4_0.OpenAPI31JSONFactory.createOpenAPI31Json(resourceDocsJson, requestedApiVersionString, hostname)
       val openApiJValue = code.api.ResourceDocs1_4_0.OpenAPI31JSONFactory.OpenAPI31JsonFormats.toJValue(openApiDoc)
 
       val yamlString = YAMLUtils.jValueToYAMLSafe(openApiJValue, "# Error converting to YAML")
-      Caching.setStaticSwaggerDocCache(cacheKey, yamlString)
+      setCache(cacheKey, yamlString)
 
       yamlString
     }
 
-    def convertResourceDocsToOpenAPI31JvalueAndSetCache(cacheKey: String, requestedApiVersionString: String, resourceDocsJson: List[JSONFactory1_4_0.ResourceDocJson]) : JValue = {
+    def convertResourceDocsToOpenAPI31JvalueAndSetCache(cacheKey: String, requestedApiVersionString: String, resourceDocsJson: List[JSONFactory1_4_0.ResourceDocJson], setCache: (String, String) => Unit = Caching.setStaticSwaggerDocCache) : JValue = {
       logger.debug(s"Generating OpenAPI 3.1-convertResourceDocsToOpenAPI31JvalueAndSetCache requestedApiVersion is $requestedApiVersionString")
       val hostname = HostName
       val openApiDoc = code.api.ResourceDocs1_4_0.OpenAPI31JSONFactory.createOpenAPI31Json(resourceDocsJson, requestedApiVersionString, hostname)
       val openApiJValue = code.api.ResourceDocs1_4_0.OpenAPI31JSONFactory.OpenAPI31JsonFormats.toJValue(openApiDoc)
 
       val jsonString = json.compactRender(openApiJValue)
-      Caching.setStaticSwaggerDocCache(cacheKey, jsonString)
+      setCache(cacheKey, jsonString)
 
       openApiJValue
     }
 
-    def convertResourceDocsToSwaggerJvalueAndSetCache(cacheKey: String, requestedApiVersionString: String,  resourceDocsJson: List[JSONFactory1_4_0.ResourceDocJson]) : JValue = {
+    def convertResourceDocsToSwaggerJvalueAndSetCache(cacheKey: String, requestedApiVersionString: String,  resourceDocsJson: List[JSONFactory1_4_0.ResourceDocJson], setCache: (String, String) => Unit = Caching.setStaticSwaggerDocCache) : JValue = {
       logger.debug(s"Generating Swagger-getResourceDocsSwaggerAndSetCache requestedApiVersion is $requestedApiVersionString")
       val swaggerDocJsonJValue = getResourceDocsSwagger(requestedApiVersionString, resourceDocsJson).head
 
       val jsonString = json.compactRender(swaggerDocJsonJValue)
-      Caching.setStaticSwaggerDocCache(cacheKey, jsonString)
+      setCache(cacheKey, jsonString)
 
       swaggerDocJsonJValue
     }

obp-api/src/main/scala/code/api/dynamic/endpoint/Http4sDynamicEndpoint.scala

@@ -127,12 +127,17 @@ object Http4sDynamicEndpoint extends MdcLoggable {
               case Some(jv) => Full(jv)
               case None     => Empty
             }
+            // Every outcome records an API metric, like any other endpoint: the handler's response
+            // under the authenticated call context (so the row names the User), and a failure to
+            // authenticate or authorise under the context the request arrived with.
             val io: IO[Response[IO]] = for {
               authedCcOpt <- IO.fromFuture(IO(doc.authCheckIO(partPath, bodyJValue, cc)))
               authedCc    = authedCcOpt.getOrElse(cc)
               resp        <- doc.dynamicHttp4sFunction.get.apply(req)(authedCc)
-            } yield resp
-            io.handleErrorWith(err => ErrorResponseConverter.toHttp4sResponse(err, cc))
+                               .handleErrorWith(err => ErrorResponseConverter.toHttp4sResponse(err, authedCc))
+              recorded    <- EndpointHelpers.recordMetricFor(resp)(authedCc)
+            } yield recorded
+            io.handleErrorWith(err => ErrorResponseConverter.toHttp4sResponse(err, cc).flatMap(EndpointHelpers.recordMetricFor(_)(cc)))
           }
         }
     }

obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala

@@ -33,7 +33,8 @@ import code.api.dynamic.endpoint.helper.practise.{DynamicEndpointCodeGenerator,
 import code.api.dynamic.endpoint.helper.practise.PractiseEndpointGroup
 import code.api.util.DynamicUtil.{DynamicCodeBody, Validation}
 import code.api.util.APIUtil.{BooleanBody, DoubleBody, EmptyBody, LongBody, Http4sEndpointIO, PrimaryDataBody, ResourceDoc, StringBody, getDisabledEndpointOperationIds}
-import code.api.util.{APIUtil, CallContext, DynamicUtil}
+import code.api.util.{APIUtil, CallContext, DynamicUtil, ErrorMessages}
+import code.api.JsonResponseException
 import net.liftweb.common.{Box, Failure, Full}
 import org.json4s.{JNothing, JValue}
 import org.json4s.JsonAST.{JBool, JDouble, JInt, JString}
@@ -111,10 +112,23 @@ object CompiledObjects {
    * validate and the dry-run compile all check against it, and CompiledObjects chooses its compiler
    * from the same normalised value, so nothing that passes the check can reach the wrong compiler.
    */
-  val supportedLanguages: List[String] = List("scala", "java")
+  val supportedLanguages: List[String] = List("scala", "java", "query")
   def isSupportedLanguage(programmingLang: String): Boolean =
     supportedLanguages.contains(APIUtil.normaliseDynamicCodeLanguage(programmingLang))
-  val supportedLanguagesText: String = "Scala, Java"
+  val supportedLanguagesText: String = "Scala, Java, Query"
+
+  /**
+   * True when the body is a Dynamic Query (`programming_lang` `Query`): a declaration, not code. It
+   * runs no user code, so the switch for user-supplied code (`allow_user_generated_scala_code`) does
+   * not apply to it, and it is read-only, so its doc's request_verb must be GET.
+   */
+  def isQuery(programmingLang: String): Boolean = APIUtil.normaliseDynamicCodeLanguage(programmingLang) == "query"
+
+  /** A Dynamic Query only reads; every other language may use any verb. */
+  def verbAllowed(programmingLang: String, requestVerb: String): Boolean =
+    !isQuery(programmingLang) || requestVerb == "GET"
+
+  val queryVerbMessage: String = "A Dynamic Query only reads, so its request_verb must be GET."
 
   /**
    * The native http4s template a method body is inlined into. Returns the full source and the
@@ -167,14 +181,26 @@ object CompiledObjects {
    * diagnostics with line numbers relative to the method body the author wrote, in the body's
    * programming language (Scala or Java). Empty = compiles.
    */
-  def compileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String, programmingLang: String = "Scala"): List[DynamicUtil.CompileProblem] = {
+  def compileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String,
+                      programmingLang: String = "Scala", bankId: Option[String] = None): List[DynamicUtil.CompileProblem] = {
     val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8")
     // Java bodies are compiled as written (no template, no generated case classes), so the example
-    // bodies play no part, as they play none in CompiledObjects' own Java branch.
-    if (APIUtil.normaliseDynamicCodeLanguage(programmingLang) == "java") DynamicUtil.checkJavaCode(decodedMethodBody)
-    else scalaCompileProblems(exampleRequestBody, successResponseBody, decodedMethodBody)
+    // bodies play no part, as they play none in CompiledObjects' own Java branch. A Dynamic Query is
+    // not compiled at all: its problems are the declaration's, which have no line numbers.
+    APIUtil.normaliseDynamicCodeLanguage(programmingLang) match {
+      case "java" => DynamicUtil.checkJavaCode(decodedMethodBody)
+      case "query" => queryProblem(decodedMethodBody, bankId).map(message => DynamicUtil.CompileProblem(0, 0, "ERROR", message)).toList
+      case _ => scalaCompileProblems(exampleRequestBody, successResponseBody, decodedMethodBody)
+    }
   }
 
+  /** What is wrong with a Dynamic Query body, in full (error code included), or None when it is valid in `bankId`'s space. */
+  def queryProblem(decodedMethodBody: String, bankId: Option[String]): Option[String] =
+    code.api.dynamic.entity.query.DynamicQueryDeclaration.parse(decodedMethodBody) match {
+      case Left(error) => Some(s"${ErrorMessages.DynamicQueryInvalid}${error.message}")
+      case Right(declaration) => code.api.dynamic.entity.query.DynamicQuery.validate(bankId, declaration).left.toOption.map(_.message)
+    }
+
   private def scalaCompileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], decodedMethodBody: String): List[DynamicUtil.CompileProblem] = {
     val requestBody: Product = exampleRequestBody match {
       case Some(JString(s)) if StringUtils.isBlank(s) => toCaseObject(None)
@@ -205,17 +231,40 @@ object CompiledObjects {
   }
 }
 
-case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String, programmingLang: String = "Scala") {
+/**
+ * `bankId` is the Dynamic Entity space of the doc (None for the system space). Only a Dynamic Query
+ * uses it: its declaration names entities, which are looked up in that space.
+ */
+case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String,
+                           programmingLang: String = "Scala", bankId: Option[String] = None) {
   val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8")
+  private val isQuery = CompiledObjects.isQuery(programmingLang)
+  // The stored doc of a system-level Dynamic Resource Doc carries Some(null) here, not None.
+  private val space: Option[String] = bankId.flatMap(Option(_)).map(_.trim).filter(_.nonEmpty)
+
+  // A Dynamic Query compiles nothing, not even case classes for its examples: toCaseObject generates
+  // and compiles Scala, which is refused where user-supplied code is switched off. Its examples are
+  // carried as JSON, which the resource-docs serialisation renders as they are.
+  private def exampleOf(json: Option[JValue]): Product =
+    if (!isQuery) toCaseObject(json)
+    else json.filter(j => j != JNothing && j != JNull).map(code.api.berlin.group.v1_3.JvalueCaseClass(_)).getOrElse(EmptyBody)
+
   val requestBody: Product = exampleRequestBody match {
       //this case means, we accept the empty string "" from json post body, we need to map it to None.
-    case Some(JString(s)) if StringUtils.isBlank(s) => toCaseObject(None)
+    case Some(JString(s)) if StringUtils.isBlank(s) => exampleOf(None)
      // Here we will generate the object by the JValue (exampleRequestBody)
-    case _ => toCaseObject(exampleRequestBody)
+    case _ => exampleOf(exampleRequestBody)
   }
-  val successResponse: Product = toCaseObject(successResponseBody)
+  val successResponse: Product = exampleOf(successResponseBody)
 
   private val partialFunction: Http4sEndpointIO = APIUtil.normaliseDynamicCodeLanguage(programmingLang) match {
+    case "query" =>
+      // A declaration, not code: parsed here (a malformed body cannot be served), checked against the
+      // entity definitions by validateDependency, and run per request by DynamicQueryEndpoint.
+      code.api.dynamic.entity.query.DynamicQueryDeclaration.parse(decodedMethodBody) match {
+        case Right(declaration) => DynamicQueryEndpoint(declaration, space)
+        case Left(error) => throw JsonResponseException(s"${ErrorMessages.DynamicQueryInvalid}${error.message}", 400, "none")
+      }
     case "java" =>
       DynamicUtil.createJavaHttp4sEndpoint(decodedMethodBody) match {
         case Full(func) => func
@@ -286,6 +335,8 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo
    */
   def validateDependency() = APIUtil.normaliseDynamicCodeLanguage(programmingLang) match {
     case "java" => ()
+    // A Dynamic Query calls no methods; what it is checked against is the entity definitions of its space.
+    case "query" => CompiledObjects.queryProblem(decodedMethodBody, space).foreach(message => throw JsonResponseException(message, 400, "none"))
     case _ => Validation.validateDependency(this.partialFunction)
   }
 
@@ -294,7 +345,7 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo
    * author wrote (the wrapper's own lines are subtracted). Empty = compiles. Nothing is evaluated or cached.
    */
   def compileProblems(): List[DynamicUtil.CompileProblem] =
-    CompiledObjects.compileProblems(exampleRequestBody, successResponseBody, methodBody, programmingLang)
+    CompiledObjects.compileProblems(exampleRequestBody, successResponseBody, methodBody, programmingLang, space)
 
   /**
    * Wraps the compiled partial function as an endpoint. This used to bind a per-bank

obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicQueryEndpoint.scala

@@ -0,0 +1,71 @@
+/**
+Open Bank Project - API
+Copyright (C) 2011-2026, TESOBE GmbH.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+  */
+
+package code.api.dynamic.endpoint.helper
+
+import cats.effect.IO
+import code.api.dynamic.entity.query.{DynamicQuery, DynamicQueryDeclaration}
+import code.api.util.APIUtil.Http4sEndpointIO
+import code.api.util.{APIUtil, CallContext}
+import code.util.Helper.MdcLoggable
+import com.openbankproject.commons.util.JsonAliases.compactRender
+import org.http4s.headers.`Content-Type`
+import org.http4s.{MediaType, Request, Response, Status}
+import org.json4s.JsonAST.{JInt, JObject, JString}
+
+/**
+ * This object turns a Dynamic Query declaration into the handler of a Dynamic Resource Doc whose
+ * `programming_lang` is `Query`. Nothing is compiled: each request is answered by
+ * [[DynamicQuery.run]] for the caller, in the Dynamic Entity space of the doc (`bankId`, None for the
+ * system space). The doc's own Roles are enforced by the middleware before this runs, as for any
+ * Dynamic Resource Doc.
+ */
+object DynamicQueryEndpoint extends MdcLoggable {
+
+  private val jsonContentType = `Content-Type`(MediaType.application.json)
+
+  def apply(declaration: DynamicQueryDeclaration, bankId: Option[String]): Http4sEndpointIO = new Http4sEndpointIO {
+    override def isDefinedAt(req: Request[IO]): Boolean = true
+
+    override def apply(req: Request[IO]): CallContext => IO[Response[IO]] = { cc =>
+      val callerParams = req.uri.query.multiParams.map { case (name, values) => name -> values.toList }
+      IO.blocking {
+        DynamicQuery.run(bankId, declaration, callerParams, cc.user.map(_.userId).toOption, APIUtil.getConsumerPrimaryKey(Some(cc)))
+      }.map {
+        case Right(result) => json(Status.Ok, result)
+        case Left(failure) => json(Status.fromInt(failure.status).getOrElse(Status.BadRequest),
+          JObject(List("code" -> JInt(failure.status), "message" -> JString(failure.message))))
+      }.handleError { e =>
+        logger.warn(s"DynamicQueryEndpoint says: the Dynamic Query from '${declaration.from}' failed", e)
+        json(Status.InternalServerError, JObject(List("code" -> JInt(500), "message" -> JString(s"OBP-50000: Unknown Error. ${e.getMessage}"))))
+      }
+    }
+  }
+
+  private def json(status: Status, body: JObject): Response[IO] =
+    Response[IO](status).withEntity(compactRender(body)).withContentType(jsonContentType)
+}