From 215ca051085d9d81c13988c364ffc555350ce8ae Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 16:43:30 +0200 Subject: [PATCH 01/20] feat: add Java language support to DynamicResourceDoc method_body can now be Scala (default, unchanged) or Java, mirroring the existing precedent already used by ConnectorMethod/DynamicMessageDoc. - add programming_lang field to JsonDynamicResourceDoc / DynamicResourceDoc.Lang - compile Java method_body via the JSR-223 java engine into a native Http4sEndpointIO, reusing the same javax.tools.JavaCompiler backend as createJavaFunction - validate the actual compiled Java class (not its Scala wrapper) against dynamic_code_compile_validate_dependencies before it is ever invoked -- the wrapper alone can't see into a compiled Function reference (typically a synthetic lambda class) the way it can for a Scala closure - convert Java-native return values (Map/List/String/number/boolean) to JValue directly, since Extraction.decompose only reflects Scala types and silently drops a raw java.util.Map's entries - reject unsupported programming_lang values with a 400 before attempting compilation - thread programming_lang through DynamicResourceDocsEndpointGroup, which recompiles every registered doc when serving it -- without this a stored Java doc would silently be recompiled as Scala and fail at request time --- .../endpoint/helper/DynamicEndpoints.scala | 21 ++- .../DynamicResourceDocsEndpointGroup.scala | 2 +- .../scala/code/api/util/DynamicUtil.scala | 132 +++++++++++++++ .../scala/code/api/v4_0_0/Http4s400.scala | 9 +- .../DynamicResourceDoc.scala | 6 +- .../DynamicResourceDocProvider.scala | 7 +- .../MappedDynamicResourceDocProvider.scala | 2 + .../src/test/resources/frozen_type_meta_data | Bin 168937 -> 168869 bytes .../DynamicUtilJavaHttp4sEndpointTest.scala | 109 +++++++++++++ .../v4_0_0/DynamicResourceDocJavaTest.scala | 151 ++++++++++++++++++ 10 files changed, 433 insertions(+), 6 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala create mode 100644 obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala index beaad62688..5d3ef7c1cc 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala @@ -77,7 +77,7 @@ trait EndpointGroup { * @param successResponseBody successResponseBody from the post json body,it is JValue here. * @param methodBody it is url-encoded string for the api level code. */ -case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String) { +case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String, programmingLang: String = "Scala") { val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8") val requestBody: Product = exampleRequestBody match { //this case means, we accept the empty string "" from json post body, we need to map it to None. @@ -87,7 +87,24 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo } val successResponse: Product = toCaseObject(successResponseBody) - private val partialFunction: Http4sEndpointIO = { + private val partialFunction: Http4sEndpointIO = programmingLang match { + case "java" | "Java" => + DynamicUtil.createJavaHttp4sEndpoint(decodedMethodBody) match { + case Full(func) => func + case Failure(msg: String, exception: Box[Throwable], _) => + throw exception.getOrElse(new RuntimeException(msg)) + case _ => throw new RuntimeException("compiled code return nothing") + } + case _ /* "Scala" | "scala" | "" | null, default */ => + scalaPartialFunction + } + + // Unchanged Scala-template compile path, factored out so the `partialFunction` match above stays + // readable. Only evaluated for Scala-language docs (the default) — Java-language docs never + // touch this, so example/response-body JValues that don't fit the Scala case-class generator + // (irrelevant for Java, since it doesn't use RequestRootJsonClass/ResponseRootJsonClass) are a + // non-issue there. + private def scalaPartialFunction: Http4sEndpointIO = { //If the requestBody is PrimaryDataBody, return None. otherwise, return the exampleRequestBody:Option[JValue] // In side OBP resourceDoc, requestBody and successResponse must be Product type, diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala index 8e6350494b..71b33776fd 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala @@ -49,7 +49,7 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel * */ private val toResourceDoc: JsonDynamicResourceDoc => ResourceDoc = { dynamicDoc => - val compiledObjects = CompiledObjects(dynamicDoc.exampleRequestBody, dynamicDoc.successResponseBody, dynamicDoc.methodBody) + val compiledObjects = CompiledObjects(dynamicDoc.exampleRequestBody, dynamicDoc.successResponseBody, dynamicDoc.methodBody, dynamicDoc.programmingLang) ResourceDoc( // partialFunction is a no-op stub — the runtime dispatch uses the native handler in // dynamicHttp4sFunction (the compiled artifact is OBPEndpointIO, not the Lift OBPEndpoint). diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 69ca8dea3c..6cbb891aed 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -44,6 +44,7 @@ object DynamicUtil extends MdcLoggable{ val toolBox: ToolBox[universe.type] = runtimeMirror(getClass.getClassLoader).mkToolBox() private val memoClassPool = new Memo[ClassLoader, ClassPool] + private val memoJavaHttp4sEndpoint = new Memo[String, Box[code.api.util.APIUtil.Http4sEndpointIO]] private def getClassPool(classLoader: ClassLoader) = memoClassPool.memoize(classLoader){ val cp = ClassPool.getDefault @@ -543,4 +544,135 @@ object DynamicUtil extends MdcLoggable{ } } } + + /** + * Converts a plain value returned by a compiled Java `method_body` into a JValue, for endpoints + * where json4s' `Extraction.decompose` cannot help: it works by Scala-case-class/collection + * reflection, so a `java.util.Map`/`java.util.List` returned from Java decomposes to `{}`/`[]` + * (its entries are invisible to Scala reflection) rather than throwing — a silent data-loss bug, + * not a compile or runtime error, so it only surfaces as an empty response body. Recurses through + * the Java collection types directly; anything else (including a Scala case class constructed + * from Java, as ConnectorMethod's Java example does) falls back to Extraction.decompose. + */ + private def javaValueToJValue(value: Any): JValue = { + import scala.jdk.CollectionConverters._ + value match { + case null => JNull + case jv: JValue => jv + case m: java.util.Map[_, _] => + JObject(m.asScala.toList.map { case (k, v) => (String.valueOf(k), javaValueToJValue(v)) }) + case l: java.util.List[_] => + JArray(l.asScala.toList.map(javaValueToJValue)) + case s: String => JString(s) + case b: java.lang.Boolean => JBool(b) + case i: java.lang.Integer => JInt(BigInt(i.intValue())) + case l: java.lang.Long => JInt(BigInt(l.longValue())) + case d: java.lang.Double => JDouble(d.doubleValue()) + case f: java.lang.Float => JDouble(f.doubleValue()) + case bd: java.math.BigDecimal => JDecimal(BigDecimal(bd)) + case other => Extraction.decompose(other)(CustomJsonFormats.formats) + } + } + + /** + * Compiles a Java `method_body` for a DynamicResourceDoc endpoint into a native + * `Http4sEndpointIO` (`PartialFunction[Request[IO], CallContext => IO[Response[IO]]]`), the same + * type the Scala template compiles to in DynamicEndpoints.CompiledObjects. + * + * Reuses the same JSR-223 "java" engine (backed by a real javax.tools.JavaCompiler via + * ch.obermuhlner:java-scriptengine — see createJavaFunction above) and the same + * package-uniquification trick, but — unlike createJavaFunction, whose DynamicFunction shape is + * specific to the ConnectorMethod feature — wraps the compiled function in a hand-written + * Http4sEndpointIO here in Scala. The Java method_body never has to construct cats.effect.IO, + * org.http4s.Response, or a Scala PartialFunction: it only ever returns a plain Java object + * (Map/List/String/number/boolean/etc.), which this adapter serializes via javaValueToJValue + * above (NOT Extraction.decompose directly — see that method's doc comment for why). + * + * Java-side convention (identical to the existing ConnectorMethod convention): the pasted class + * implements java.util.function.Supplier>. The + * compiled function is invoked with: + * args(0) = the raw request body (String, or null if the request had none) + * args(1) = path params (java.util.Map) + * args(2) = the CallContext (present whenever this endpoint is actually being served) + * mirroring createJavaFunction's own `func(args ++ cc)` call (line above): appending an + * Option[CallContext] via `++` appends its *contents* (0 or 1 raw CallContext), not the Option + * wrapper itself, so Java reads args[2] directly as a CallContext, no unwrapping needed. + * + * Unlike createJavaFunction, this validates the actual compiled Java class (not just its Scala + * wrapper) against `dynamic_code_compile_validate_dependencies`/`dynamic_code_compile_validate_enable`. + * CompiledObjects.validateDependency() (called by the ResourceDoc-creation flow) only ever sees + * `this.partialFunction` — the hand-written Http4sEndpointIO below — whose own bytecode just + * calls `java.util.function.Function.apply`, a non-restricted type; it can't see what the pasted + * Java class does inside apply(Object[]). Worse, `func` itself (the Function returned by the + * pasted class's get()) is commonly a method reference (`this::apply`), which the JVM + * materialises as a synthetic lambda class whose bytecode is just a delegating call — validating + * `func.getClass` would be equally blind. So we go through the JSR-223 Compilable API directly + * (JavaScriptEngine implements it) instead of plain eval(), to get the real top-level compiled + * class/instance (JavaCompiledScript.getCompiledClass/getCompiledInstance) and validate that + * before the function is ever returned or invoked. + */ + def createJavaHttp4sEndpoint(methodBody: String): Box[code.api.util.APIUtil.Http4sEndpointIO] = + if (!dynamicCodeExecutionEnabled) Failure(ErrorMessages.DynamicCodeExecutionDisabled) + else memoJavaHttp4sEndpoint.memoize("java-http4s-endpoint:" + methodBody) { + import cats.effect.IO + import code.api.util.APIUtil.Http4sEndpointIO + import com.openbankproject.commons.ExecutionContext.Implicits.global + import com.openbankproject.commons.util.JsonAliases.compactRender + import org.http4s.headers.`Content-Type` + import org.http4s.dsl.io._ + import org.http4s.{MediaType, Request, Response} + + import scala.jdk.CollectionConverters._ + + Box tryo { + val packageExp = UUID.randomUUID().toString.replaceAll("^|-", "_") + val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) + + val javaCode = s"""package code.api.util.dynamic.${packageExp}; + |${packageMatcher.replaceFirst("")} + |""".stripMargin + + // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — + // any Java syntax/type error surfaces as an exception, caught by the outer `Box tryo`. + val compiledScript = javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) + .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] + + // Validate the real compiled Supplier class before it's ever invoked — see the doc comment + // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`. + Validation.validateDependency(compiledScript.getCompiledInstance) + + val func = compiledScript.eval().asInstanceOf[java.util.function.Function[Array[AnyRef], Any]] + val jsonContentType = `Content-Type`(MediaType.application.json) + + new Http4sEndpointIO { + override def isDefinedAt(req: Request[IO]): Boolean = true + + override def apply(req: Request[IO]): CallContext => IO[Response[IO]] = { cc => + val pathParams: java.util.Map[String, String] = cc.resourceDocument + .map(_.getPathParams(req.uri.path.segments.toList.map(_.encoded))) + .getOrElse(Map.empty[String, String]) + .asJava + + val valueIO: IO[Any] = IO.fromFuture(IO { + Future { + val args: Array[AnyRef] = Array(cc.httpBody.orNull, pathParams) + func(args ++ Some(cc)) + } + }) + + valueIO.flatMap { value => + Ok(compactRender(javaValueToJValue(value)), jsonContentType) + }.handleErrorWith { e => + logger.warn(s"createJavaHttp4sEndpoint: Java method_body threw", e) + InternalServerError( + compactRender(Extraction.decompose( + Map("code" -> 500, "message" -> s"OBP-50000: Unknown Error. ${e.getMessage}") + )(CustomJsonFormats.formats)), + jsonContentType + ) + } + } + } + } + } } diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index d821649e0d..8aaa8cc810 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -9374,12 +9374,19 @@ object Http4s400 { case _ => true } } + // Fail fast with a clean 400 before attempting compilation, rather than surfacing an + // unsupported programming_lang only as a generic DynamicCodeCompileFail. + _ <- code.util.Helper.booleanToFuture( + s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: Scala, Java""", + cc = Some(cc)) { + Set("", "scala", "Scala", "java", "Java").contains(body.programmingLang) + } } yield () } private def compileDynamicResourceDoc(body: JsonDynamicResourceDoc, cc: CallContext): Unit = { try { - CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody).validateDependency() + CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency() } catch { case e: JsonResponseException => throw e case e: Exception => diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala index a96c164350..b46485383e 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala @@ -25,6 +25,9 @@ class DynamicResourceDoc extends LongKeyedMapper[DynamicResourceDoc] with IdPK w object Tags extends MappedString(this, 255) object Roles extends MappedString(this, 255) object MethodBody extends MappedText(this) + // Source language of MethodBody: "Scala" (default) or "Java". Mirrors DynamicMessageDoc.Lang / + // ConnectorMethod.programmingLang — same field name/width convention, see DynamicEndpoints. + object Lang extends MappedString(this, 50) // Provenance: who created / last updated this runtime-compiled endpoint, and a SHA-256 of the // (decoded) method body so tampering / drift is detectable. Set server-side from the CallContext // user — never from the request body. createdAt / updatedAt come from the CreatedUpdated trait. @@ -50,7 +53,8 @@ object DynamicResourceDoc extends DynamicResourceDoc with LongKeyedMetaMapper[Dy successResponseBody = Option(dynamicResourceDoc.SuccessResponseBody.get).filter(StringUtils.isNotBlank).map(json.parse), errorResponseBodies = dynamicResourceDoc.ErrorResponseBodies.get, tags = dynamicResourceDoc.Tags.get, - roles = dynamicResourceDoc.Roles.get + roles = dynamicResourceDoc.Roles.get, + programmingLang = dynamicResourceDoc.Lang.get ) } diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala index 60e94a243a..aca89c7f76 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala @@ -33,7 +33,12 @@ case class JsonDynamicResourceDoc( successResponseBody: Option[JValue], errorResponseBodies: String, tags: String, - roles: String + roles: String, + // Source language of methodBody: "Scala" (default) or "Java". Mirrors + // JsonConnectorMethod.programmingLang / JsonDynamicMessageDoc.programmingLang. Appended last + // (not inserted alphabetically) so existing named-arg call sites and JSON payloads that predate + // this field keep compiling/deserializing unchanged. + programmingLang: String = "Scala" ) extends JsonFieldReName { def decodedMethodBody: String = URLDecoder.decode(methodBody, "UTF-8") } diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala index 47be5d0442..7b8d25a700 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala @@ -82,6 +82,7 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { .Tags(entity.tags) .Roles(entity.roles) .MethodBody(entity.methodBody) + .Lang(entity.programmingLang) // provenance is set here from the authenticated user + computed hash, not from `entity` .CreatedByUserId(createdByUserId.getOrElse(null)) .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) @@ -107,6 +108,7 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { .Tags(entity.tags) .Roles(entity.roles) .MethodBody(entity.methodBody) + .Lang(entity.programmingLang) // CreatedByUserId is left untouched; record who last changed the code + refresh the hash .UpdatedByUserId(updatedByUserId.getOrElse(null)) .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) diff --git a/obp-api/src/test/resources/frozen_type_meta_data b/obp-api/src/test/resources/frozen_type_meta_data index a15d8f19007c90d68212eb29be282ccc4714a6df..a49f5f72b3abe876dbae42c5566d78b45b1f5810 100644 GIT binary patch delta 180 zcmaF4oNMWFt_?Gd7=5Q}`!ULG-eHu>KmGb-M*Zm*#F@A^+c_;XW)z#8cvg6NdZs54pN zsOaSLRVvf}@G?nFeiylM`=lF;<>HKD+qph4GI27_pL{W2a=Y43#t4JyPePeYr~9!n Ya&K4GVtQl%H11#>qwsdgXr>Dm0IgL*BLDyZ delta 202 zcmZ3woa^Ost_?Gd82zUE_%X_D-eHu>KRMfyZ@R88BlqTbr-jCh!kZb(k`x(*Co`65 zZ|BShjorWDGHwynYeaWUt%2Y+jBIAkwk7C+q0RZ** BNhtsT diff --git a/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala b/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala new file mode 100644 index 0000000000..ab984a317b --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala @@ -0,0 +1,109 @@ +package code.api.util + +import cats.effect.IO +import cats.effect.unsafe.implicits.global +import org.http4s.{Method, Request, Uri} +import org.json4s.native.JsonMethods.parse +import org.scalatest.{FeatureSpec, GivenWhenThen, Matchers} + +/** + * Focused unit test for DynamicUtil.createJavaHttp4sEndpoint, isolated from the full + * register -> role-check -> HTTP-dispatch round trip (see DynamicResourceDocJavaTest for that). + * Exercises the adapter directly: compiled Java Supplier> -> + * Http4sEndpointIO.apply(Request[IO]) -> CallContext => IO[Response[IO]]. + */ +class DynamicUtilJavaHttp4sEndpointTest extends FeatureSpec with Matchers with GivenWhenThen { + + private val echoMethodBody = + """package code.api.util.dynamic; + | + |import code.api.util.CallContext; + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaHttp4sEndpointUnitTest implements Supplier> { + | private Object apply(Object[] args) { + | String rawBody = (String) args[0]; + | @SuppressWarnings("unchecked") + | Map pathParams = (Map) args[1]; + | CallContext cc = (CallContext) args[2]; + | + | Map response = new LinkedHashMap<>(); + | response.put("echoed_body", rawBody); + | response.put("path_param_count", pathParams.size()); + | response.put("correlation_id", cc.correlationId()); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + feature("DynamicUtil.createJavaHttp4sEndpoint compiles a Java method_body into a native Http4sEndpointIO") { + + scenario("the compiled endpoint reads args(0)/args(1)/args(2) and serves 200 JSON") { + Given("a Java method_body compiled via createJavaHttp4sEndpoint") + val endpoint = DynamicUtil.createJavaHttp4sEndpoint(echoMethodBody).openOrThrowException("compilation failed") + + When("the compiled endpoint handles a request carrying a body, no path params, and a CallContext") + val req = Request[IO](method = Method.POST, uri = Uri.unsafeFromString("/test")) + val cc = CallContext(httpBody = Some("""{"hello":"world"}"""), correlationId = "test-correlation-id") + val resp = endpoint.apply(req)(cc).unsafeRunSync() + + Then("the response is 200 and echoes the body, the (empty) path params, and the CallContext's correlationId") + resp.status.code should equal(200) + val bodyString = resp.body.through(fs2.text.utf8.decode).compile.string.unsafeRunSync() + val json = parse(bodyString) + (json \ "echoed_body").values should equal("""{"hello":"world"}""") + (json \ "path_param_count").values should equal(BigInt(0)) + (json \ "correlation_id").values should equal("test-correlation-id") + } + + scenario("a Java compile error is reported as a Box Failure, not a thrown exception") { + Given("a method_body that is not valid Java") + val badMethodBody = "this is not valid java at all" + + When("we try to compile it") + val result = DynamicUtil.createJavaHttp4sEndpoint(badMethodBody) + + Then("compilation fails gracefully") + result.isDefined should equal(false) + } + + scenario("a Java method_body that throws at runtime is recovered as a 500, not an uncaught exception") { + Given("a Java method_body whose apply() throws") + val throwingMethodBody = + """package code.api.util.dynamic; + | + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaHttp4sEndpointThrowingTest implements Supplier> { + | private Object apply(Object[] args) { + | throw new RuntimeException("boom"); + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + val endpoint = DynamicUtil.createJavaHttp4sEndpoint(throwingMethodBody).openOrThrowException("compilation failed") + + When("the compiled endpoint is invoked") + val req = Request[IO](method = Method.POST, uri = Uri.unsafeFromString("/test")) + val resp = endpoint.apply(req)(CallContext()).unsafeRunSync() + + Then("the response is 500 rather than the IO failing") + resp.status.code should equal(500) + val bodyString = resp.body.through(fs2.text.utf8.decode).compile.string.unsafeRunSync() + bodyString should include("boom") + } + } +} diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala new file mode 100644 index 0000000000..34cb2cedee --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala @@ -0,0 +1,151 @@ +package code.api.v4_0_0 + +import org.json4s._ +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, DynamicCodeLangNotSupport, UserHasMissingRoles} +import code.dynamicResourceDoc.JsonDynamicResourceDoc +import code.entitlement.Entitlement +import com.openbankproject.commons.model.ErrorMessage +import com.openbankproject.commons.util.json +import org.json4s.native.JsonMethods.{compact, parse => parseJson, render} +import org.json4s.native.Serialization.write + +/** + * Java-language coverage for the DynamicResourceDoc runtime-compilation mechanism. + * DynamicResourceDocTest.scala covers the (unchanged) Scala-language path end-to-end; these + * scenarios exercise the new `programming_lang = "Java"` dispatch added to + * DynamicEndpoints.CompiledObjects / DynamicUtil.createJavaHttp4sEndpoint, plus the + * backward-compat and unsupported-language guards added alongside it. + */ +class DynamicResourceDocJavaTest extends V400ServerSetup { + + // Java-side convention: the pasted class implements Supplier>. + // args(0) = raw request body (String, or null), args(1) = path params (java.util.Map), + // args(2) = the CallContext. See DynamicUtil.createJavaHttp4sEndpoint's doc comment. + private def javaRoleTestMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaResourceDocRoleTest implements Supplier> { + | private Object apply(Object[] args) { + | String rawBody = (String) args[0]; + | @SuppressWarnings("unchecked") + | Map pathParams = (Map) args[1]; + | String myUserId = pathParams.get("MY_USER_ID"); + | + | Map response = new LinkedHashMap<>(); + | response.put("user_id_from_path", myUserId + "_from_path"); + | response.put("received_body", rawBody); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + feature("Native execution of a runtime-compiled dynamic resource doc with a Java method_body") { + + scenario("Create a role-gated Java-language dynamic resource doc and verify 401 / 403 / 200") { + val dynamicRole = "CanCallJavaPieceCRoleTest" + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a Java-language dynamic resource doc gated by that role") + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = dynamicRole, + partialFunctionName = "javaPieceCRoleTest", + requestUrl = "/my_java_role_user/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(javaRoleTestMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val createResp = makePostRequest(createReq, write(doc)) + Then("We should get a 201") + createResp.code should equal(201) + createResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Java") + + val callUrl = dynamicEndpoint_Request / "dynamic-resource-doc" / "my_java_role_user" / "user-1" + val body = """{"name":"Jhon","age":12,"hobby":["coding"]}""" + + Then("calling without authentication returns 401") + val resp401 = makePostRequest(callUrl.POST, body) + resp401.code should equal(401) + resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) + + Then("calling authenticated but without the role returns 403") + val resp403 = makePostRequest(callUrl.POST <@ (user1), body) + resp403.code should equal(403) + resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) + + Then("granting the role makes the call succeed (200), served by the compiled Java class") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) + val resp200 = makePostRequest(callUrl.POST <@ (user1), body) + resp200.code should equal(200) + val rendered = json.compactRender(resp200.body) + rendered should include("user-1_from_path") + rendered should include("Jhon") + } + + scenario("Reject an unsupported programming_lang before attempting compilation") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a dynamic resource doc with an unsupported programming_lang") + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + partialFunctionName = "unsupportedLangTest", + requestUrl = "/unsupported_lang_test/MY_USER_ID", + programmingLang = "Python" + ) + val resp = makePostRequest(createReq, write(doc)) + + Then("We should get a 400 DynamicCodeLangNotSupport, not a compile-failure error") + resp.code should equal(400) + resp.body.extract[ErrorMessage].message should include(DynamicCodeLangNotSupport) + } + + scenario("Backward compatibility: a request body with programming_lang entirely omitted still creates a Scala-language doc") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a dynamic resource doc from a JSON payload that predates the programming_lang field") + val posted = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "preExistingClientTest", + requestUrl = "/pre_existing_client_test/MY_USER_ID" + ) + // Simulate an old client payload: strip programming_lang out entirely rather than relying on + // Serialization.write, which always emits every case-class field (default or not). + val fullJson = parseJson(write(posted)) + val withoutLang = fullJson.removeField { case (name, _) => name == "programming_lang" } + val requestBodyStr = compact(render(withoutLang)) + requestBodyStr should not include "programming_lang" + + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val createResp = makePostRequest(createReq, requestBodyStr) + + Then("We should get a 201 and the stored/served doc defaults to the Scala language") + createResp.code should equal(201) + createResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Scala") + + Then("calling the endpoint still compiles and serves via the (unchanged) Scala template path") + val callReq = (dynamicEndpoint_Request / "dynamic-resource-doc" / "pre_existing_client_test" / "user-1").POST <@ (user1) + val callResp = makePostRequest(callReq, """{"name":"Jhon","age":12,"hobby":["coding"]}""") + callResp.code should equal(200) + val rendered = json.compactRender(callResp.body) + rendered should include("user-1_from_path") + rendered should include("Jhon") + } + } +} From 69eaa44fddec063780b6a59a8140529b563ccaf7 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 16:43:36 +0200 Subject: [PATCH 02/20] fix: thread programming_lang into dynamic-resource-doc validate endpoint POST /management/dynamic-resource-docs/validate constructed CompiledObjects without the doc's programming_lang, so validating a Java-language method_body silently tried to compile it as Scala and always reported a (misleading) CompilationError, and the success message hardcoded "valid Scala" regardless of the actual language. --- obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 3b91ef8fff..aeae1a0f39 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -4358,10 +4358,10 @@ object Http4s600 { } } yield try { code.api.dynamic.endpoint.helper.CompiledObjects( - body.exampleRequestBody, body.successResponseBody, body.methodBody).validateDependency() + body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency() ValidateDynamicResourceDocSuccessJsonV600( valid = true, - message = "Dynamic Resource Doc method body is valid Scala and uses allowed dependencies.") + message = s"Dynamic Resource Doc method body is valid ${body.programmingLang} and uses allowed dependencies.") } catch { case e: code.api.JsonResponseException => val errorText = e.jsonResponse match { From 75dff0495da82640e46ab78e0b202acc48cf921e Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 17:19:10 +0200 Subject: [PATCH 03/20] test: extract duplicated dynamic-resource-docs request literal to a helper SonarCloud flagged the "dynamic-resource-docs" URL segment repeated three times across DynamicResourceDocJavaTest's three scenarios. --- .../code/api/v4_0_0/DynamicResourceDocJavaTest.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala index 34cb2cedee..8ce6a30af0 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala @@ -20,6 +20,8 @@ import org.json4s.native.Serialization.write */ class DynamicResourceDocJavaTest extends V400ServerSetup { + private def createDynamicResourceDocsRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + // Java-side convention: the pasted class implements Supplier>. // args(0) = raw request body (String, or null), args(1) = path params (java.util.Map), // args(2) = the CallContext. See DynamicUtil.createJavaHttp4sEndpoint's doc comment. @@ -58,7 +60,7 @@ class DynamicResourceDocJavaTest extends V400ServerSetup { Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) When("We create a Java-language dynamic resource doc gated by that role") - val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val createReq = createDynamicResourceDocsRequest val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( dynamicResourceDocId = None, bankId = None, @@ -99,7 +101,7 @@ class DynamicResourceDocJavaTest extends V400ServerSetup { Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) When("We create a dynamic resource doc with an unsupported programming_lang") - val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val createReq = createDynamicResourceDocsRequest val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( dynamicResourceDocId = None, bankId = None, @@ -132,7 +134,7 @@ class DynamicResourceDocJavaTest extends V400ServerSetup { val requestBodyStr = compact(render(withoutLang)) requestBodyStr should not include "programming_lang" - val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val createReq = createDynamicResourceDocsRequest val createResp = makePostRequest(createReq, requestBodyStr) Then("We should get a 201 and the stored/served doc defaults to the Scala language") From cd1b548469ade7562896e2403f923f19e3ce0338 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 17:24:45 +0200 Subject: [PATCH 04/20] test: extract shared 401/403/200 role-gate assertion for dynamic resource docs DynamicResourceDocJavaTest's role-gated scenario duplicated the same 17-line 401/403/200 assertion block already in DynamicResourceDocTest, which SonarCloud's new-code duplication gate flagged. Both now share V400ServerSetup.assertRoleGated401Then403Then200. --- .../v4_0_0/DynamicResourceDocJavaTest.scala | 24 +++++------------ .../api/v4_0_0/DynamicResourceDocTest.scala | 20 +++----------- .../code/api/v4_0_0/V400ServerSetup.scala | 27 ++++++++++++++++++- 3 files changed, 36 insertions(+), 35 deletions(-) diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala index 8ce6a30af0..aa12c85bc8 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala @@ -3,7 +3,7 @@ package code.api.v4_0_0 import org.json4s._ import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON import code.api.util.ApiRole -import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, DynamicCodeLangNotSupport, UserHasMissingRoles} +import code.api.util.ErrorMessages.DynamicCodeLangNotSupport import code.dynamicResourceDoc.JsonDynamicResourceDoc import code.entitlement.Entitlement import com.openbankproject.commons.model.ErrorMessage @@ -78,23 +78,11 @@ class DynamicResourceDocJavaTest extends V400ServerSetup { val callUrl = dynamicEndpoint_Request / "dynamic-resource-doc" / "my_java_role_user" / "user-1" val body = """{"name":"Jhon","age":12,"hobby":["coding"]}""" - Then("calling without authentication returns 401") - val resp401 = makePostRequest(callUrl.POST, body) - resp401.code should equal(401) - resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) - - Then("calling authenticated but without the role returns 403") - val resp403 = makePostRequest(callUrl.POST <@ (user1), body) - resp403.code should equal(403) - resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) - - Then("granting the role makes the call succeed (200), served by the compiled Java class") - Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) - val resp200 = makePostRequest(callUrl.POST <@ (user1), body) - resp200.code should equal(200) - val rendered = json.compactRender(resp200.body) - rendered should include("user-1_from_path") - rendered should include("Jhon") + assertRoleGated401Then403Then200(callUrl, body, dynamicRole) { resp200 => + val rendered = json.compactRender(resp200.body) + rendered should include("user-1_from_path") + rendered should include("Jhon") + } } scenario("Reject an unsupported programming_lang before attempting compilation") { diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala index a482557a18..ec60ccb3b9 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala @@ -29,7 +29,7 @@ import org.json4s._ import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON import code.api.util.APIUtil.OAuth._ import code.api.util.ApiRole._ -import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, DynamicResourceDocAlreadyExists, DynamicResourceDocNotFound, UserHasMissingRoles} +import code.api.util.ErrorMessages.{DynamicResourceDocAlreadyExists, DynamicResourceDocNotFound, UserHasMissingRoles} import code.api.util.ApiRole import code.api.v4_0_0.APIMethods400.Implementations4_0_0 import code.dynamicResourceDoc.JsonDynamicResourceDoc @@ -315,21 +315,9 @@ class DynamicResourceDocTest extends V400ServerSetup { val callUrl = dynamicEndpoint_Request / "dynamic-resource-doc" / "my_role_user" / "user-1" val body = """{"name":"Jhon","age":12,"hobby":["coding"]}""" - Then("calling without authentication returns 401") - val resp401 = makePostRequest(callUrl.POST, body) - resp401.code should equal(401) - resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) - - Then("calling authenticated but without the role returns 403") - val resp403 = makePostRequest(callUrl.POST <@ (user1), body) - resp403.code should equal(403) - resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) - - Then("granting the role makes the call succeed (200)") - Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) - val resp200 = makePostRequest(callUrl.POST <@ (user1), body) - resp200.code should equal(200) - json.compactRender(resp200.body) should include("_from_path") + assertRoleGated401Then403Then200(callUrl, body, dynamicRole) { resp200 => + json.compactRender(resp200.body) should include("_from_path") + } } // Regression guard for DynamicEndpointCodeGenerator.buildTemplate: the template served by diff --git a/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala b/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala index ba69e99330..e33eee4eb5 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala @@ -21,9 +21,10 @@ import code.metadata.comments.MappedComment import code.metadata.narrative.MappedNarrative import code.metadata.transactionimages.MappedTransactionImage import code.metadata.wheretags.MappedWhereTag +import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, UserHasMissingRoles} import code.setup.{APIResponse, DefaultUsers, ServerSetupWithTestData} import code.transactionattribute.MappedTransactionAttribute -import com.openbankproject.commons.model.{AccountId, AccountRoutingJsonV121, AmountOfMoneyJsonV121, BankId, CreateViewJson, UpdateViewJSON} +import com.openbankproject.commons.model.{AccountId, AccountRoutingJsonV121, AmountOfMoneyJsonV121, BankId, CreateViewJson, ErrorMessage, UpdateViewJSON} import com.openbankproject.commons.util.ApiShortVersions import code.setup.OBPReq import org.json4s.native.Serialization.write @@ -42,6 +43,30 @@ trait V400ServerSetup extends ServerSetupWithTestData with DefaultUsers { def dynamicEndpoint_Request: OBPReq = baseRequest / "obp" / ApiShortVersions.`dynamic-endpoint`.toString def dynamicEntity_Request: OBPReq = baseRequest / "obp" / ApiShortVersions.`dynamic-entity`.toString + /** + * Shared by DynamicResourceDocTest and DynamicResourceDocJavaTest: exercises + * ResourceDoc.authCheckIO's role-gated path against a runtime-compiled dynamic-resource-doc -- + * calling without auth returns 401, authenticated without the role returns 403, and granting the + * role makes the call succeed (200), handed to `assertSuccess` for endpoint-specific checks. + */ + def assertRoleGated401Then403Then200(callUrl: OBPReq, body: String, dynamicRole: String)(assertSuccess: APIResponse => Unit): Unit = { + Then("calling without authentication returns 401") + val resp401 = makePostRequest(callUrl.POST, body) + resp401.code should equal(401) + resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) + + Then("calling authenticated but without the role returns 403") + val resp403 = makePostRequest(callUrl.POST <@ (user1), body) + resp403.code should equal(403) + resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) + + Then("granting the role makes the call succeed (200)") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) + val resp200 = makePostRequest(callUrl.POST <@ (user1), body) + resp200.code should equal(200) + assertSuccess(resp200) + } + def randomBankId : String = { def getBanksInfo : APIResponse = { val request = v4_0_0_Request / "banks" From 2a754c7456c67baef8ce871a6cdefecc5add7eef Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 17:35:38 +0200 Subject: [PATCH 05/20] test: regenerate frozen_type_meta_data.txt to match the updated blob Kept in sync with frozen_type_meta_data (regenerated by the previous commit) via code.util.FrozenMetaDataText -- FrozenMetaDataTextTest fails when the two disagree. --- obp-api/src/test/resources/frozen_type_meta_data.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/obp-api/src/test/resources/frozen_type_meta_data.txt b/obp-api/src/test/resources/frozen_type_meta_data.txt index d7bddd02ce..bf2252ef5b 100644 --- a/obp-api/src/test/resources/frozen_type_meta_data.txt +++ b/obp-api/src/test/resources/frozen_type_meta_data.txt @@ -511,7 +511,6 @@ field code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON.SeverJWK use String field code.api.util.APIUtil.BooleanBody value Boolean field code.api.util.APIUtil.EndpointInfo name String field code.api.util.APIUtil.EndpointInfo version String -field code.api.util.APIUtil.JArrayBody value org.json4s.JArray field code.api.v1_2_1.APIInfoJSON connector String field code.api.v1_2_1.APIInfoJSON git_commit String field code.api.v1_2_1.APIInfoJSON hosted_by code.api.v1_2_1.HostedBy @@ -3145,6 +3144,7 @@ field code.dynamicResourceDoc.JsonDynamicResourceDoc errorResponseBodies String field code.dynamicResourceDoc.JsonDynamicResourceDoc exampleRequestBody Option[org.json4s.JValue] field code.dynamicResourceDoc.JsonDynamicResourceDoc methodBody String field code.dynamicResourceDoc.JsonDynamicResourceDoc partialFunctionName String +field code.dynamicResourceDoc.JsonDynamicResourceDoc programmingLang String field code.dynamicResourceDoc.JsonDynamicResourceDoc requestUrl String field code.dynamicResourceDoc.JsonDynamicResourceDoc requestVerb String field code.dynamicResourceDoc.JsonDynamicResourceDoc roles String From 3a50efdc55b8269cb4f3f8b62f91f4809c191f4d Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 18:15:26 +0200 Subject: [PATCH 06/20] fix: make Java dynamic-code dependency validation actually enforce Two bugs made createJavaHttp4sEndpoint's validation a no-op in practice, discovered by manually enabling dynamic_code_compile_validate_enable against a real Java method_body that calls a non-whitelisted OBP method: - Javassist's LoaderClassPath reads a class's bytecode via classLoader.getResourceAsStream(...), but the compiler's ch.obermuhlner.scriptengine.java.MemoryClassLoader only overrides loadClass() and never exposes compiled bytes as a classpath resource. getDynamicCodeDependentMethods silently found nothing to check no matter what the Java code called. Fix: read the bytes from the classloader's private byte map and register them with Javassist directly via ByteArrayClassPath. - Once dependency lookup worked, a rejection was still swallowed: JsonResponseException never sets a Throwable message, so wrapping the validation call in Box.tryo turned it into Failure(null, Full(theException), Empty), and DynamicEndpoints.scala's `case Failure(msg: String, ...)` pattern fails to match a null msg -- silently falling through to a generic "compiled code return nothing" error and discarding the real rejection reason. Fix: run validation outside any Box.tryo, mirroring how the Scala path's CompiledObjects.validateDependency() (also never tryo-wrapped) already lets the exception propagate uncaught. Manually verified end-to-end: a Java method_body calling code.api.util.APIUtil.getPropsValue (not on the dependency whitelist) is now rejected with 400 OBP-40046, naming the exact forbidden call. --- .../scala/code/api/util/DynamicUtil.scala | 39 +++++++++++++++++-- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 6cbb891aed..98392f7dd4 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -624,7 +624,11 @@ object DynamicUtil extends MdcLoggable{ import scala.jdk.CollectionConverters._ - Box tryo { + // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — any + // Java syntax/type error surfaces as an exception, caught by this `Box tryo` and turned into + // a Failure. Deliberately narrow: only the compile step is wrapped. Validation runs below, + // outside any tryo — see the comment there for why that separation matters. + val compiledScriptBox: Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript] = Box tryo { val packageExp = UUID.randomUUID().toString.replaceAll("^|-", "_") val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) @@ -632,10 +636,37 @@ object DynamicUtil extends MdcLoggable{ |${packageMatcher.replaceFirst("")} |""".stripMargin - // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — - // any Java syntax/type error surfaces as an exception, caught by the outer `Box tryo`. - val compiledScript = javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) + javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] + } + + // Deliberately outside compiledScriptBox's `Box tryo`: a rejection here throws + // JsonResponseException, which must propagate UNCAUGHT (mirroring the Scala path's + // CompiledObjects.validateDependency(), also never wrapped in tryo) so + // compileDynamicResourceDoc's `case e: JsonResponseException => throw e` sees it intact. + // JsonResponseException never sets a Throwable message (getMessage == null); Box.tryo would + // catch it into Failure(null, Full(theException), Empty), and DynamicEndpoints.scala's + // `case Failure(msg: String, ...)` pattern silently fails to match a null msg — falling + // through to "compiled code return nothing" and discarding the real rejection reason. `.map` + // does not swallow exceptions the way `Box tryo` does, so this stays uncaught here. + compiledScriptBox.map { compiledScript => + val compiledClass = compiledScript.getCompiledClass + + // getDynamicCodeDependentMethods loads the class's bytecode via Javassist's LoaderClassPath, + // which reads it through classLoader.getResourceAsStream(...). The compiler's + // ch.obermuhlner.scriptengine.java.MemoryClassLoader only overrides loadClass() — it never + // exposes the compiled bytes as a classpath resource — so that lookup silently fails + // (javassist.NotFoundException) and validation would see zero dependent methods no matter + // what the Java code actually calls. Read the bytes directly from the classloader's private + // byte map (reflection is unavoidable here: java-scriptengine exposes no public accessor) + // and hand them to Javassist explicitly via ByteArrayClassPath, so the real method bodies — + // including any restricted OBP call — are visible to validation. + val classBytesField = compiledClass.getClassLoader.getClass.getDeclaredField("mapClassBytes") + classBytesField.setAccessible(true) + val classBytes = classBytesField.get(compiledClass.getClassLoader) + .asInstanceOf[java.util.Map[String, Array[Byte]]].get(compiledClass.getName) + getClassPool(compiledClass.getClassLoader) + .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes)) // Validate the real compiled Supplier class before it's ever invoked — see the doc comment // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`. From 6ce26699a3a441e110a8ecd859ffcce5c4487a4c Mon Sep 17 00:00:00 2001 From: Hongwei Date: Mon, 31 Aug 2026 23:04:39 +0200 Subject: [PATCH 07/20] test: cover Java dynamic-code dependency validation end-to-end DynamicUtil.Validation's dependency whitelist and Constant.SHOW_USED_CONNECTOR_METHODS were final vals, computed once when first touched (in practice during server boot, before any test scenario runs). setPropsValues -- the standard per-scenario props override used throughout this test suite -- mutates Props.lockedProviders, which only reaches code that re-reads Props on each call; it cannot un-freeze an already-computed field. That made this security path untestable without a real server restart, which is how the earlier manual verification (see prior commit) had to prove it. Changed the whole SHOW_USED_CONNECTOR_METHODS / Validation dependency chain from val to def so it re-evaluates from live props on each call. This costs nothing extra in production: the one expensive step, DynamicUtil.compileScalaCodeUnchecked, is already memoized by the exact source string, so re-evaluating is a cache hit unless the underlying props value actually changed. DynamicResourceDocJavaSecurityValidationTest exercises the resulting create-time HTTP path directly: a Java method_body calling code.api.util.APIUtil.getPropsValue (not on the dependency whitelist) is rejected with 400 OBP-40046, naming the exact forbidden call -- the same assertion the manual verification made by hand. --- .../scala/code/api/constant/constant.scala | 7 +- .../scala/code/api/util/DynamicUtil.scala | 25 ++++--- ...esourceDocJavaSecurityValidationTest.scala | 70 +++++++++++++++++++ 3 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index d4bfed683b..fa51de2af9 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -305,7 +305,12 @@ object Constant extends MdcLoggable { final val CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL: Int = APIUtil.getPropsValue(s"createLocalisedResourceDocJson.cache.ttl.seconds", "3600").toInt final val GET_DYNAMIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"dynamicResourceDocsObp.cache.ttl.seconds", "3600").toInt final val GET_STATIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"staticResourceDocsObp.cache.ttl.seconds", "3600").toInt - final val SHOW_USED_CONNECTOR_METHODS: Boolean = APIUtil.getPropsAsBoolValue(s"show_used_connector_methods", false) + // def, not final val: DynamicUtil.Validation.validateDependency (dynamic-code dependency + // checking) needs this to react to a props change without a restart -- e.g. test-time + // setPropsValues overrides. A final val here would freeze at whatever value was true the + // moment this object was first touched (typically during server boot, well before any test + // scenario runs), and no later prop override could ever reach it. + def SHOW_USED_CONNECTOR_METHODS: Boolean = APIUtil.getPropsAsBoolValue(s"show_used_connector_methods", false) // Rate Limiting Cache Prefixes (with global namespace and versioning) // Both call_counter and rl_active are versioned for consistent cache invalidation diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 98392f7dd4..42a1537263 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -361,10 +361,17 @@ object DynamicUtil extends MdcLoggable{ object Validation { - val dynamicCodeSandboxPermissions = APIUtil.getPropsValue("dynamic_code_sandbox_permissions", "[]").trim - val scalaCodePermissioins = "List[java.security.Permission]"+dynamicCodeSandboxPermissions.replaceFirst("\\[","(").dropRight(1)+")" - val permissions:Box[List[java.security.Permission]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodePermissioins) - + // def, not val, throughout this object: these must react to a props change (e.g. test-time + // setPropsValues) without a restart, not freeze at whatever the props held the moment + // Validation was first touched (typically by whichever dynamic-code test happens to run + // first in a shared test JVM). This costs nothing extra in production -- the only expensive + // step, DynamicUtil.compileScalaCodeUnchecked, is already memoized by the exact source + // string, so re-evaluating these on every call is a cache hit unless the underlying props + // value actually changed. + def dynamicCodeSandboxPermissions = APIUtil.getPropsValue("dynamic_code_sandbox_permissions", "[]").trim + def scalaCodePermissioins = "List[java.security.Permission]"+dynamicCodeSandboxPermissions.replaceFirst("\\[","(").dropRight(1)+")" + def permissions:Box[List[java.security.Permission]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodePermissioins) + // all Permissions put at here // Here is the Java Permission document, please extend these permissions carefully. // https://docs.oracle.com/javase/8/docs/technotes/guides/security/spec/security-spec.doc3.html#17001 @@ -383,15 +390,15 @@ object DynamicUtil extends MdcLoggable{ // new RuntimePermission("accessDeclaredMembers"), // new RuntimePermission("getClassLoader"), // ) - val allowedRuntimePermissions = permissions.openOrThrowException("Can not compile the props `dynamic_code_sandbox_permissions` to permissions") + def allowedRuntimePermissions = permissions.openOrThrowException("Can not compile the props `dynamic_code_sandbox_permissions` to permissions") - val dependenciesString = APIUtil.getPropsValue("dynamic_code_compile_validate_dependencies", "[]").trim + def dependenciesString = APIUtil.getPropsValue("dynamic_code_compile_validate_dependencies", "[]").trim // `Map[String, String](` rather than `Map(`: the props default is an empty list, and a bare // `Map()` leaves its type parameters undetermined, so the trailing .toMap cannot prove the // elements are pairs and the reflective compilation fails. The .toMap itself is needed because // mapValues returns a view rather than a Map. - val scalaCodeDependencies = s"${DynamicUtil.importStatements}"+dependenciesString.replaceFirst("\\[","Map[String, String](").dropRight(1) +").mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet).toMap" - val dependenciesBox: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodeDependencies) + def scalaCodeDependencies = s"${DynamicUtil.importStatements}"+dependenciesString.replaceFirst("\\[","Map[String, String](").dropRight(1) +").mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet).toMap" + def dependenciesBox: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodeDependencies) /** * Compilation OBP Dependencies Guard, only checked the OBP methods, not scala/Java libraies(are checked during the runtime.). @@ -422,7 +429,7 @@ object DynamicUtil extends MdcLoggable{ // PractiseEndpoint.getClass.getTypeName + "*" -> "*", // // ).mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet) - val allowedCompilationMethods: Map[String, Set[String]] = dependenciesBox.openOrThrowException("Can not compile the props `dynamic_code_compile_validate_dependencies` to Map") + def allowedCompilationMethods: Map[String, Set[String]] = dependenciesBox.openOrThrowException("Can not compile the props `dynamic_code_compile_validate_dependencies` to Map") //Do not touch this Set, try to use the `allowedPermissions` and `allowedMethods` to control the sandbox val restrictedTypes = Set( diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala new file mode 100644 index 0000000000..e54d7a5f19 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -0,0 +1,70 @@ +package code.api.v4_0_0 + +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.api.util.ErrorMessages.DynamicResourceDocMethodDependency +import code.entitlement.Entitlement +import com.openbankproject.commons.model.ErrorMessage +import org.json4s.native.Serialization.write + +/** + * With dynamic_code_compile_validate_enable=true, a Java method_body that calls an OBP method NOT + * on the dependency whitelist must be rejected -- proving createJavaHttp4sEndpoint validates the + * real compiled Java class (getCompiledInstance), not just its Scala wrapper. + */ +class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { + + private def maliciousMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaSecurityProbe implements Supplier> { + | private Object apply(Object[] args) { + | // APIUtil.getPropsValue is NOT on dynamic_code_compile_validate_dependencies' + | // whitelist (only errorJsonResponse*/scalaFutureToLaFuture/futureToBoxedResponse are). + | String secret = code.api.util.APIUtil$.MODULE$.getPropsValue("hostname", "none"); + | Map response = new LinkedHashMap<>(); + | response.put("leaked", secret); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + feature("Security validation of Java method_body against dynamic_code_compile_validate_dependencies") { + scenario("Registering a Java doc that calls a non-whitelisted OBP method is rejected with 400") { + setPropsValues( + "show_used_connector_methods" -> "true", + "dynamic_code_compile_validate_enable" -> "true", + "dynamic_code_compile_validate_dependencies" -> + """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[Sandbox].getTypeName -> "runInSandbox", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" + ) + + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "securityProbeTest", + requestUrl = "/security_probe_test/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(maliciousMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val resp = makePostRequest(createReq, write(doc)) + + Then("the compile is rejected with 400 DynamicResourceDocMethodDependency, not accepted") + resp.code should equal(400) + resp.body.extract[ErrorMessage].message should include(DynamicResourceDocMethodDependency) + } + } +} From 9b458b6d85cebe60537f8ebb3bf1f38e23abbc07 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 08:38:18 +0200 Subject: [PATCH 08/20] fix: don't let a cached compile result skip fresh dependency validation createJavaHttp4sEndpoint memoized the whole Box[Http4sEndpointIO] -- including whatever Validation.validateDependency decided -- keyed only by the exact method_body string. Once dynamic_code_compile_validate_enable and the whitelist became live-reloadable (previous commit), this became a bypass: compile a Java source while validation is off, then turn validation on and tighten the whitelist, then resubmit the identical source in a new create/update call -- the cached Full(...) from the first, unvalidated compile is returned directly, and validateDependency never runs a second time. Split the memoization so only the actual javax.tools.JavaCompiler invocation is cached (deterministic given the source, and the one genuinely expensive step); dependency validation now runs on every call, unmemoized, exactly like the Scala path's CompiledObjects.validateDependency() already does. Also corrected an overclaiming comment: the val-to-def change makes Validation.allowedRuntimePermissions itself always current, but Sandbox.sandbox(bankId) separately caches the whole Sandbox it builds per bankId, so a sandbox built before a permissions-prop change keeps the old snapshot. Left that cache alone -- SecurityManager enforcement is already a no-op on this JDK (JEP 486), so neither the stale nor the fresh permission list is actually enforced either way. DynamicResourceDocJavaSecurityValidationTest gains a regression scenario reproducing the exact bypass sequence: compile the malicious source once with validation off, enable strict validation, resubmit the identical source under a new doc, and assert it is still rejected. --- .../scala/code/api/util/DynamicUtil.scala | 70 +++++++++++++------ ...esourceDocJavaSecurityValidationTest.scala | 61 +++++++++++++--- 2 files changed, 100 insertions(+), 31 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 42a1537263..6c37de021b 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -44,7 +44,9 @@ object DynamicUtil extends MdcLoggable{ val toolBox: ToolBox[universe.type] = runtimeMirror(getClass.getClassLoader).mkToolBox() private val memoClassPool = new Memo[ClassLoader, ClassPool] - private val memoJavaHttp4sEndpoint = new Memo[String, Box[code.api.util.APIUtil.Http4sEndpointIO]] + // Caches only the compiled artifact (deterministic given the source string), never the + // validation outcome built on top of it -- see createJavaHttp4sEndpoint's doc comment. + private val memoJavaCompiledScript = new Memo[String, Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript]] private def getClassPool(classLoader: ClassLoader) = memoClassPool.memoize(classLoader){ val cp = ClassPool.getDefault @@ -368,6 +370,14 @@ object DynamicUtil extends MdcLoggable{ // step, DynamicUtil.compileScalaCodeUnchecked, is already memoized by the exact source // string, so re-evaluating these on every call is a cache hit unless the underlying props // value actually changed. + // + // This makes allowedRuntimePermissions itself always current, but NOT everything downstream + // of it: Sandbox.sandbox(bankId) below separately caches the whole Sandbox it builds, keyed + // only by bankId -- so a bankId whose sandbox was already built keeps that snapshot of + // allowedRuntimePermissions until the process restarts, same staleness this def change fixed + // for validateDependency. Left as-is here because it's moot in practice: SecurityManager + // enforcement is already a no-op on this JVM (JEP 486, JDK 24+; see Sandbox's own comment), + // so neither the stale nor the fresh permission list is actually enforced. def dynamicCodeSandboxPermissions = APIUtil.getPropsValue("dynamic_code_sandbox_permissions", "[]").trim def scalaCodePermissioins = "List[java.security.Permission]"+dynamicCodeSandboxPermissions.replaceFirst("\\[","(").dropRight(1)+")" def permissions:Box[List[java.security.Permission]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodePermissioins) @@ -620,7 +630,7 @@ object DynamicUtil extends MdcLoggable{ */ def createJavaHttp4sEndpoint(methodBody: String): Box[code.api.util.APIUtil.Http4sEndpointIO] = if (!dynamicCodeExecutionEnabled) Failure(ErrorMessages.DynamicCodeExecutionDisabled) - else memoJavaHttp4sEndpoint.memoize("java-http4s-endpoint:" + methodBody) { + else { import cats.effect.IO import code.api.util.APIUtil.Http4sEndpointIO import com.openbankproject.commons.ExecutionContext.Implicits.global @@ -631,25 +641,38 @@ object DynamicUtil extends MdcLoggable{ import scala.jdk.CollectionConverters._ - // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — any - // Java syntax/type error surfaces as an exception, caught by this `Box tryo` and turned into - // a Failure. Deliberately narrow: only the compile step is wrapped. Validation runs below, - // outside any tryo — see the comment there for why that separation matters. - val compiledScriptBox: Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript] = Box tryo { - val packageExp = UUID.randomUUID().toString.replaceAll("^|-", "_") - val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) - - val javaCode = s"""package code.api.util.dynamic.${packageExp}; - |${packageMatcher.replaceFirst("")} - |""".stripMargin - - javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) - .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] - } + // Only the compile step is memoized — deterministic given the same source string, and the + // one genuinely expensive part (a real javax.tools.JavaCompiler invocation). Dependency + // validation below is NOT memoized: it depends on mutable external config + // (dynamic_code_compile_validate_enable/_dependencies), which can change between two + // createJavaHttp4sEndpoint calls for the identical source string — e.g. a doc compiled once + // while validation was off, then a later create/update call resubmitting the exact same + // method_body after validation was turned on and the whitelist tightened. An earlier version + // of this function memoized the validated *result* (Box[Http4sEndpointIO]) as a single unit, + // so that second call silently reused the first call's unvalidated success — bypassing the + // now-stricter policy for any resubmitted source. Re-running validation on every call costs + // little: it is Javassist bytecode inspection plus a Map lookup, not another compile. + val compiledScriptBox: Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript] = + memoJavaCompiledScript.memoize("java-http4s-endpoint:" + methodBody) { + // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — + // any Java syntax/type error surfaces as an exception, caught by this `Box tryo` and + // turned into a Failure. + Box tryo { + val packageExp = UUID.randomUUID().toString.replaceAll("^|-", "_") + val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) + + val javaCode = s"""package code.api.util.dynamic.${packageExp}; + |${packageMatcher.replaceFirst("")} + |""".stripMargin + + javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) + .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] + } + } - // Deliberately outside compiledScriptBox's `Box tryo`: a rejection here throws - // JsonResponseException, which must propagate UNCAUGHT (mirroring the Scala path's - // CompiledObjects.validateDependency(), also never wrapped in tryo) so + // Deliberately outside compiledScriptBox's `Box tryo` AND outside the memoization above: a + // rejection here throws JsonResponseException, which must propagate UNCAUGHT (mirroring the + // Scala path's CompiledObjects.validateDependency(), also never wrapped in tryo) so // compileDynamicResourceDoc's `case e: JsonResponseException => throw e` sees it intact. // JsonResponseException never sets a Throwable message (getMessage == null); Box.tryo would // catch it into Failure(null, Full(theException), Empty), and DynamicEndpoints.scala's @@ -667,7 +690,9 @@ object DynamicUtil extends MdcLoggable{ // what the Java code actually calls. Read the bytes directly from the classloader's private // byte map (reflection is unavoidable here: java-scriptengine exposes no public accessor) // and hand them to Javassist explicitly via ByteArrayClassPath, so the real method bodies — - // including any restricted OBP call — are visible to validation. + // including any restricted OBP call — are visible to validation. Re-registering the same + // bytes on a compile-cache hit is harmless: Javassist's ClassPool just gains a duplicate, + // already-matching ClassPath entry. val classBytesField = compiledClass.getClassLoader.getClass.getDeclaredField("mapClassBytes") classBytesField.setAccessible(true) val classBytes = classBytesField.get(compiledClass.getClassLoader) @@ -676,7 +701,8 @@ object DynamicUtil extends MdcLoggable{ .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes)) // Validate the real compiled Supplier class before it's ever invoked — see the doc comment - // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`. + // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`, + // and why it must run fresh on every call rather than being cached with the compile result. Validation.validateDependency(compiledScript.getCompiledInstance) val func = compiledScript.eval().asInstanceOf[java.util.function.Function[Array[AnyRef], Any]] diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala index e54d7a5f19..c7ceaae1ab 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -39,18 +39,24 @@ class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { |} |""".stripMargin + // Mirrors sample.props.template's default dynamic_code_compile_validate_dependencies exactly, + // minus the trailing newlines/line-continuations -- deliberately does NOT list APIUtil.getPropsValue. + private def defaultDependenciesWhitelist: String = + """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[Sandbox].getTypeName -> "runInSandbox", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" + + private def enableStrictValidation(): Unit = setPropsValues( + "show_used_connector_methods" -> "true", + "dynamic_code_compile_validate_enable" -> "true", + "dynamic_code_compile_validate_dependencies" -> defaultDependenciesWhitelist + ) + + private def createRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + feature("Security validation of Java method_body against dynamic_code_compile_validate_dependencies") { scenario("Registering a Java doc that calls a non-whitelisted OBP method is rejected with 400") { - setPropsValues( - "show_used_connector_methods" -> "true", - "dynamic_code_compile_validate_enable" -> "true", - "dynamic_code_compile_validate_dependencies" -> - """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[Sandbox].getTypeName -> "runInSandbox", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" - ) - + enableStrictValidation() Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) - val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( dynamicResourceDocId = None, bankId = None, @@ -60,11 +66,48 @@ class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { methodBody = java.net.URLEncoder.encode(maliciousMethodBody, "UTF-8"), programmingLang = "Java" ) - val resp = makePostRequest(createReq, write(doc)) + val resp = makePostRequest(createRequest, write(doc)) Then("the compile is rejected with 400 DynamicResourceDocMethodDependency, not accepted") resp.code should equal(400) resp.body.extract[ErrorMessage].message should include(DynamicResourceDocMethodDependency) } + + // Regression guard for the bug createJavaHttp4sEndpoint had before it split compilation from + // validation: memoJavaCompiledScript (formerly memoJavaHttp4sEndpoint) memoized the WHOLE + // Box[Http4sEndpointIO], keyed only by the exact method_body string. A doc compiled once while + // validation was off got a cached Full(...) that a later, identical create call -- made AFTER + // validation was turned on and the whitelist tightened -- would silently reuse, never + // re-running Validation.validateDependency at all. This scenario reproduces exactly that + // sequence: compile the same malicious source once with validation off (succeeds, populates + // the compile cache), then enable strict validation and resubmit the identical source. + scenario("A Java source compiled once while validation was off is still validated on a later create call") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val firstDoc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "cacheBypassProbeTest1", + requestUrl = "/cache_bypass_probe_test_1/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(maliciousMethodBody, "UTF-8"), + programmingLang = "Java" + ) + When("validation is off (the suite default) and we compile the malicious source for the first time") + val firstResp = makePostRequest(createRequest, write(firstDoc)) + firstResp.code should equal(201) + + When("validation is then turned on with the same source resubmitted under a different doc") + enableStrictValidation() + val secondDoc = firstDoc.copy( + partialFunctionName = "cacheBypassProbeTest2", + requestUrl = "/cache_bypass_probe_test_2/MY_USER_ID" + ) + val secondResp = makePostRequest(createRequest, write(secondDoc)) + + Then("the second create is still rejected -- the compile-result cache must not bypass fresh validation") + secondResp.code should equal(400) + secondResp.body.extract[ErrorMessage].message should include(DynamicResourceDocMethodDependency) + } } } From a369957d8bcfded5a365cdfa3af5ea074f18ec5e Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 08:41:03 +0200 Subject: [PATCH 09/20] fix: satisfy CI's test-isolation lint for the setPropsValues helper enableStrictValidation() wrapped setPropsValues with an expression body (def ... = setPropsValues(...)), which .github/scripts/ check_test_isolation.py's brace-based scanner does not recognize as a safe "helper" scope -- it only classifies a def as safe when it finds an opening brace immediately following the def name. The call was flagged as running at class-instantiation time and failed CI's lint step before compilation even started. Verified locally with python3 .github/scripts/check_test_isolation.py. --- ...micResourceDocJavaSecurityValidationTest.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala index c7ceaae1ab..3e056158c7 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -44,11 +44,16 @@ class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { private def defaultDependenciesWhitelist: String = """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[Sandbox].getTypeName -> "runInSandbox", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" - private def enableStrictValidation(): Unit = setPropsValues( - "show_used_connector_methods" -> "true", - "dynamic_code_compile_validate_enable" -> "true", - "dynamic_code_compile_validate_dependencies" -> defaultDependenciesWhitelist - ) + // Block body (not `= setPropsValues(...)`) so .github/scripts/check_test_isolation.py's brace + // scanner sees an opening `{` right after `def enableStrictValidation` and treats this as a + // safe "helper called from scenarios" scope rather than a class-body-level setPropsValues call. + private def enableStrictValidation(): Unit = { + setPropsValues( + "show_used_connector_methods" -> "true", + "dynamic_code_compile_validate_enable" -> "true", + "dynamic_code_compile_validate_dependencies" -> defaultDependenciesWhitelist + ) + } private def createRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) From b7bef716036d7f241006aed03e7d8c0b2a836bdf Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 09:09:12 +0200 Subject: [PATCH 10/20] fix: decouple dependency-whitelist validation from show_used_connector_methods getDynamicCodeDependentMethods and APIUtil.getDependentMethods were both gated by SHOW_USED_CONNECTOR_METHODS, an unrelated introspection/reporting toggle that defaults to false. This meant dynamic_code_compile_validate_enable alone did nothing: with the reporting prop left at its default, the bytecode scan always returned an empty dependency list, so every dynamic-code call passed the whitelist check regardless of what it actually invoked. Add a force parameter that Validation.validateDependency sets to true, making security validation controlled solely by dynamic_code_compile_validate_enable as documented. The existing caller in DynamicCompileEndpoint keeps its prior behaviour unchanged. --- .../main/scala/code/api/util/APIUtil.scala | 6 +++-- .../scala/code/api/util/DynamicUtil.scala | 27 ++++++++++++++----- ...esourceDocJavaSecurityValidationTest.scala | 8 +++++- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 9e45f49432..e591a362bd 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -4399,8 +4399,10 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * * than the return value may be (getUserAndSessionContextFuture, ***,***),(map,***,***), (getOrElse,***,***) ...... */ - def getDependentMethods(className: String, methodName:String, signature: String): List[(String, String, String)] = { - if (SHOW_USED_CONNECTOR_METHODS) { + // force bypasses the SHOW_USED_CONNECTOR_METHODS gate below -- see + // DynamicUtil.getDynamicCodeDependentMethods' doc comment for why security validation needs this. + def getDependentMethods(className: String, methodName:String, signature: String, force: Boolean = false): List[(String, String, String)] = { + if (SHOW_USED_CONNECTOR_METHODS || force) { val methods = ListBuffer[(String, String, String)]() //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. //eg: className == code.api.UKOpenBanking.v3_1_0.APIMethods_AccountAccessApi$$anonfun$createAccountAccessConsents$lzycompute$1 diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 6c37de021b..173bdca707 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -170,26 +170,37 @@ object DynamicUtil extends MdcLoggable{ } /** - * NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. + * NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. * @param clazz * @param predicate + * @param force bypasses the SHOW_USED_CONNECTOR_METHODS gate below. SHOW_USED_CONNECTOR_METHODS + * exists to opt in to an unrelated, expensive introspection/reporting feature (which + * connector methods a static endpoint touches) — it was never meant to gate SECURITY + * validation, which reuses this same bytecode scan. Without `force`, a deployment + * that sets dynamic_code_compile_validate_enable=true (the documented, security- + * relevant prop) but leaves the unrelated show_used_connector_methods at its default + * false would silently get an always-empty dependency list here — every dynamic-code + * call looks "allowed" no matter what it does, because there is nothing to check + * against the whitelist. Validation.validateDependency passes force=true so it is + * controlled solely by dynamic_code_compile_validate_enable, matching what an + * operator following that prop's own documentation would expect. * @return */ - def getDynamicCodeDependentMethods(clazz: Class[_], predicate: String => Boolean = _ => true): List[(String, String, String)] = - if (SHOW_USED_CONNECTOR_METHODS) { + def getDynamicCodeDependentMethods(clazz: Class[_], predicate: String => Boolean = _ => true, force: Boolean = false): List[(String, String, String)] = + if (SHOW_USED_CONNECTOR_METHODS || force) { val className = clazz.getTypeName val listBuffer = new ListBuffer[(String, String, String)]() val classPool = getClassPool(clazz.getClassLoader) - //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. + //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. val ctClass = classPool.get(className) for { method <- ctClass.getDeclaredMethods.toList if predicate(method.getName) - ternary @ (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature) + ternary @ (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature, force) } yield { // if method is also dynamic compile code, extract it's dependent method if(className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName+ "$")) { - listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature)) + listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature, force)) } else { listBuffer.append(ternary) } @@ -478,7 +489,9 @@ object DynamicUtil extends MdcLoggable{ def validateDependency(obj: AnyRef): Unit = { if(APIUtil.getPropsAsBoolValue("dynamic_code_compile_validate_enable",false)){ - val dependentMethods: List[(String, String, String)] = DynamicUtil.getDynamicCodeDependentMethods(obj.getClass) + // force=true: this check must not also require the unrelated show_used_connector_methods + // prop -- see getDynamicCodeDependentMethods' doc comment for why. + val dependentMethods: List[(String, String, String)] = DynamicUtil.getDynamicCodeDependentMethods(obj.getClass, force = true) validateDependency(dependentMethods) } else{ // If false, nothing to do here. ; diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala index 3e056158c7..66fabae06b 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -44,12 +44,18 @@ class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { private def defaultDependenciesWhitelist: String = """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[Sandbox].getTypeName -> "runInSandbox", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" + // Deliberately does NOT set show_used_connector_methods: that prop exists to opt in to an + // unrelated, expensive introspection/reporting feature and was never meant to gate security + // validation, which happens to reuse the same underlying bytecode scan. An operator who reads + // only dynamic_code_compile_validate_enable's own prop documentation and sets just these two + // props (as this method does) must still get real enforcement -- proving that is the point of + // every scenario below. + // // Block body (not `= setPropsValues(...)`) so .github/scripts/check_test_isolation.py's brace // scanner sees an opening `{` right after `def enableStrictValidation` and treats this as a // safe "helper called from scenarios" scope rather than a class-body-level setPropsValues call. private def enableStrictValidation(): Unit = { setPropsValues( - "show_used_connector_methods" -> "true", "dynamic_code_compile_validate_enable" -> "true", "dynamic_code_compile_validate_dependencies" -> defaultDependenciesWhitelist ) From eee46acd848ab3574ecce0d942ac2bc99d81c297 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 09:10:09 +0200 Subject: [PATCH 11/20] fix: register compiled Java bytecode with ClassPool only once per source createJavaHttp4sEndpoint re-registered the compiled class's bytes into the shared Javassist ClassPool on every call, including compile-cache hits. ClassPool.appendClassPath has no dedup, so a long-running process serving the same dynamic endpoint repeatedly (e.g. on each resourceDocs-list rebuild) grew the ClassPool's classpath chain without bound. Move the registration inside the compile memoization block so it runs exactly once per distinct source string, tied to the same cache lifetime as the compiled class itself. --- .../scala/code/api/util/DynamicUtil.scala | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 173bdca707..fee661611b 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -678,8 +678,32 @@ object DynamicUtil extends MdcLoggable{ |${packageMatcher.replaceFirst("")} |""".stripMargin - javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) + val compiledScript = javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] + + // getDynamicCodeDependentMethods loads a class's bytecode via Javassist's + // LoaderClassPath, which reads it through classLoader.getResourceAsStream(...). The + // compiler's ch.obermuhlner.scriptengine.java.MemoryClassLoader only overrides + // loadClass() — it never exposes the compiled bytes as a classpath resource — so that + // lookup silently fails (javassist.NotFoundException) and validation would see zero + // dependent methods no matter what the Java code actually calls. Read the bytes + // directly from the classloader's private byte map (reflection is unavoidable here: + // java-scriptengine exposes no public accessor) and hand them to Javassist explicitly + // via ByteArrayClassPath, so the real method bodies — including any restricted OBP + // call — are visible to validation. Done here, inside the compile memoization, so it + // runs exactly once per distinct source: ClassPool.appendClassPath has no dedup of its + // own, so doing this on every createJavaHttp4sEndpoint call (as an earlier version of + // this function did, on every resourceDocs-list rebuild for the process's lifetime) + // grew that ClassPool's classpath chain without bound. + val compiledClass = compiledScript.getCompiledClass + val classBytesField = compiledClass.getClassLoader.getClass.getDeclaredField("mapClassBytes") + classBytesField.setAccessible(true) + val classBytes = classBytesField.get(compiledClass.getClassLoader) + .asInstanceOf[java.util.Map[String, Array[Byte]]].get(compiledClass.getName) + getClassPool(compiledClass.getClassLoader) + .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes)) + + compiledScript } } @@ -693,26 +717,6 @@ object DynamicUtil extends MdcLoggable{ // through to "compiled code return nothing" and discarding the real rejection reason. `.map` // does not swallow exceptions the way `Box tryo` does, so this stays uncaught here. compiledScriptBox.map { compiledScript => - val compiledClass = compiledScript.getCompiledClass - - // getDynamicCodeDependentMethods loads the class's bytecode via Javassist's LoaderClassPath, - // which reads it through classLoader.getResourceAsStream(...). The compiler's - // ch.obermuhlner.scriptengine.java.MemoryClassLoader only overrides loadClass() — it never - // exposes the compiled bytes as a classpath resource — so that lookup silently fails - // (javassist.NotFoundException) and validation would see zero dependent methods no matter - // what the Java code actually calls. Read the bytes directly from the classloader's private - // byte map (reflection is unavoidable here: java-scriptengine exposes no public accessor) - // and hand them to Javassist explicitly via ByteArrayClassPath, so the real method bodies — - // including any restricted OBP call — are visible to validation. Re-registering the same - // bytes on a compile-cache hit is harmless: Javassist's ClassPool just gains a duplicate, - // already-matching ClassPath entry. - val classBytesField = compiledClass.getClassLoader.getClass.getDeclaredField("mapClassBytes") - classBytesField.setAccessible(true) - val classBytes = classBytesField.get(compiledClass.getClassLoader) - .asInstanceOf[java.util.Map[String, Array[Byte]]].get(compiledClass.getName) - getClassPool(compiledClass.getClassLoader) - .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes)) - // Validate the real compiled Supplier class before it's ever invoked — see the doc comment // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`, // and why it must run fresh on every call rather than being cached with the compile result. From 04b1417ada0d964830c310c1a09dfdf65bab0b58 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 09:11:59 +0200 Subject: [PATCH 12/20] fix: fall back to Scala for dynamic resource docs with a NULL lang column getJsonDynamicResourceDoc passed Lang.get straight through as a constructor argument. Rows created before the Lang column existed have a genuine SQL NULL there, and an explicit null argument bypasses JsonDynamicResourceDoc's own "Scala" default -- that default only applies when the argument is omitted. Legacy rows therefore reported programming_lang as null/empty instead of the documented default. Fall back explicitly with Option(...).filter(isNotBlank).getOrElse("Scala"), matching the pattern already used for the other nullable text fields in the same method. Added a regression test that forces a genuine NULL via raw SQL (bypassing the ORM, which always writes "Scala") and asserts GET still reports "Scala". --- .../DynamicResourceDoc.scala | 6 +++- .../api/v4_0_0/DynamicResourceDocTest.scala | 36 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala index b46485383e..d42ef429fd 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala @@ -54,7 +54,11 @@ object DynamicResourceDoc extends DynamicResourceDoc with LongKeyedMetaMapper[Dy errorResponseBodies = dynamicResourceDoc.ErrorResponseBodies.get, tags = dynamicResourceDoc.Tags.get, roles = dynamicResourceDoc.Roles.get, - programmingLang = dynamicResourceDoc.Lang.get + // Rows created before the Lang column existed have NULL there, not "Scala" -- a bare + // Lang.get would surface that as an empty/null programming_lang instead of falling back to + // JsonDynamicResourceDoc's own "Scala" default, since an explicit null argument bypasses a + // case class default (that only applies when the argument is omitted entirely). + programmingLang = Option(dynamicResourceDoc.Lang.get).filter(StringUtils.isNotBlank).getOrElse("Scala") ) } diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala index ec60ccb3b9..c7d21954af 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala @@ -414,6 +414,42 @@ class DynamicResourceDocTest extends V400ServerSetup { storedRow.UpdatedByUserId.get should be(resourceUser1.userId) storedRow.MethodBodyHash.get should be(code.api.util.APIUtil.sha256Hex(URLDecoder.decode(changedMethodBody, "UTF-8"))) } + + // Regression guard: rows created before the Lang column existed have a genuine SQL NULL there + // (Schemifier's ALTER TABLE ADD COLUMN sets no default), not "Scala". An explicit null argument + // bypasses JsonDynamicResourceDoc's own programmingLang="Scala" default -- that default only + // applies when the argument is omitted entirely -- so a bare Lang.get would have surfaced as a + // null/empty programming_lang in the API response instead of falling back to "Scala". + scenario("A dynamic resource doc row predating the programming_lang column still reports \"Scala\"", ApiEndpoint1, ApiEndpoint3, VersionOfApi) { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canGetDynamicResourceDoc.toString) + + When("We create a dynamic resource doc") + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val posted = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + partialFunctionName = "preDatesLangColumnTest", + requestUrl = "/pre_dates_lang_column_test/MY_USER_ID" + ) + val createResp = makePostRequest(createReq, write(posted)) + createResp.code should equal(201) + val docId = (createResp.body \ "dynamic_resource_doc_id").values.toString + + When("its lang column is forced to a genuine SQL NULL, bypassing the ORM (which always writes \"Scala\")") + import code.dynamicResourceDoc.DynamicResourceDoc + net.liftweb.mapper.DB.runUpdate( + s"UPDATE ${DynamicResourceDoc.dbTableName} SET ${DynamicResourceDoc.Lang.dbColumnName} = NULL " + + s"WHERE ${DynamicResourceDoc.DynamicResourceDocId.dbColumnName} = ?", + List(docId) + ) + + Then("GET still reports programming_lang as \"Scala\", not null or empty") + val getReq = (v4_0_0_Request / "management" / "dynamic-resource-docs" / docId).GET <@ (user1) + val getResp = makeGetRequest(getReq) + getResp.code should equal(200) + getResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Scala") + } } } From 759cc825d810c7db7e1e487cbd584110fd2e866c Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 10:36:02 +0200 Subject: [PATCH 13/20] fix: strict dependency validation unconditionally rejected every Java dynamic resource doc Two compounding bugs, both required to reproduce and both required to fix, found by running the feature against a real packaged jar and a real Postgres database instead of the in-process test harness: 1. Every Java method_body implements Supplier> per convention. javac always erases that generic Supplier.get() to a synthetic bridge method (Object get()) whose body just invokevirtual-calls the real, properly-typed get() -- an ordinary same-class call. The dependency scanner's same-class exemption only recognised a Scala-specific mangled-name convention, so this call was treated as a dependency on a forbidden method: the dynamically-compiled class lives under the OBP-owned code.* package, but its randomly-generated name can never appear in a static whitelist. 2. CompiledObjects.validateDependency() re-validated this.partialFunction a second time after DynamicUtil.createJavaHttp4sEndpoint had already validated the real compiled Java class internally. For Java, this.partialFunction is OBP's own Http4sEndpointIO wrapper, not user code, and its bytecode legitimately calls internal OBP helpers (javaValueToJValue, logger, CustomJsonFormats.formats, compactRender) that were never meant to be whitelisted. Together these meant dynamic_code_compile_validate_enable=true rejected every Java doc unconditionally, benign or not -- invisible to the existing test suite because no scenario combined a genuinely benign Java body with strict validation turned on. Fix getDynamicCodeDependentMethods to recurse through any same-class self-call rather than treat it as a leaf dependency, and make CompiledObjects.validateDependency() a no-op for the Java language (the real validation already happened inside createJavaHttp4sEndpoint). --- .../endpoint/helper/DynamicEndpoints.scala | 15 +++++- .../scala/code/api/util/DynamicUtil.scala | 15 +++++- ...esourceDocJavaSecurityValidationTest.scala | 50 +++++++++++++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala index 5d3ef7c1cc..97b5c34459 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala @@ -174,8 +174,21 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo * this will check all the dynamic scala code dependencies at compile time. * *Search for the usage, you can see how to use it in OBP code. + * + * Scala-only: for the Scala language, `this.partialFunction` IS the compiled user code, so + * validating its bytecode directly is correct. For Java, `this.partialFunction` is instead + * OBP's own Http4sEndpointIO wrapper (built by DynamicUtil.createJavaHttp4sEndpoint) around the + * real compiled Java class -- its bytecode legitimately calls internal OBP helpers + * (DynamicUtil.javaValueToJValue/logger, CustomJsonFormats.formats, JsonAliases.compactRender) + * that were never meant to be dependency-whitelisted, since they are framework glue, not + * user-supplied code. createJavaHttp4sEndpoint already validates the real compiled Java class + * internally (see its own doc comment) before ever returning that wrapper, so re-validating the + * wrapper here is both redundant and wrong -- it would reject every Java doc unconditionally. */ - def validateDependency() = Validation.validateDependency(this.partialFunction) + def validateDependency() = programmingLang match { + case "java" | "Java" => () + case _ => Validation.validateDependency(this.partialFunction) + } /** * This is used to check the security permission at the run time. diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index fee661611b..f572fe55bd 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -199,7 +199,20 @@ object DynamicUtil extends MdcLoggable{ ternary @ (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature, force) } yield { // if method is also dynamic compile code, extract it's dependent method - if(className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName+ "$")) { + if (typeName == className) { + // A same-class self-call is not itself a dependency to police -- recurse into what the + // target method calls instead of flagging the call as forbidden. This is required for + // Java: every Java dynamic resource doc implements Supplier> + // (the documented convention), and the compiler always erases that generic Supplier.get() + // to a synthetic bridge method `Object get()` whose body is just `return this.get();` -- + // an ordinary same-class invokevirtual call to the real, properly-typed get(). Without this + // branch that bridge call is scanned like any other, resolves to (thisClass, "get", ...), + // and since the dynamically-compiled class lives under the OBP-owned code.* package + // (isRestrictedType) but its randomly-generated name can never appear in a static + // whitelist, EVERY Java doc is unconditionally rejected under strict validation -- + // regardless of what its code actually does. + listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature, force)) + } else if(className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName+ "$")) { listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature, force)) } else { listBuffer.append(ternary) diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala index 66fabae06b..72f6cc6934 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -61,9 +61,59 @@ class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { ) } + // Every Java method_body implements Supplier> per convention (see + // DynamicUtil.createJavaHttp4sEndpoint's doc comment). javac always erases that generic + // Supplier.get() to a synthetic bridge method `Object get()` whose body just invokevirtual-calls + // the real, properly-typed get() -- an ordinary same-class call regardless of what the body does. + private def benignMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaSecurityBenignProbe implements Supplier> { + | private Object apply(Object[] args) { + | Map response = new LinkedHashMap<>(); + | response.put("greeting", "hello"); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + private def createRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) feature("Security validation of Java method_body against dynamic_code_compile_validate_dependencies") { + + // Regression guard: the Supplier.get() generics-erasure bridge method's same-class call to the + // real get() must not itself be treated as a call to a forbidden method. Without this, every + // Java doc -- malicious or not -- was rejected under strict validation, because the compiled + // class lives under the OBP-owned code.* package but its randomly-generated name can never + // appear in a static whitelist. + scenario("Registering a benign Java doc succeeds even with strict validation enabled") { + enableStrictValidation() + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "benignProbeTest", + requestUrl = "/benign_probe_test/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(benignMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val resp = makePostRequest(createRequest, write(doc)) + + Then("the compile succeeds -- the Supplier.get() bridge method's self-call is not a forbidden dependency") + resp.code should equal(201) + } scenario("Registering a Java doc that calls a non-whitelisted OBP method is rejected with 400") { enableStrictValidation() Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) From adea5e5c25240f2d34644875c98ed6f6d48fd2f3 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 16:39:54 +0200 Subject: [PATCH 14/20] fix: recurse through multi-level same-class calls in Java dependency scan The Supplier.get() bridge-method exemption added for Java dynamic resource docs only unrolled one level: it appended a same-class method's own callees as raw dependency tuples instead of recursing into them. A Java method_body that factors logic into its own private helper methods (fully ordinary Java) reintroduces the exact same false rejection one level deeper -- the un-recursed helper's callees get flagged as calls to the compiled class's own randomly-generated, unwhitelistable name. Replace the one-level unroll with a proper recursive expansion that follows same-class calls to arbitrary depth until a genuinely foreign dependency is reached, with a visited-set cycle guard: direct or mutual recursion between same-class private methods (e.g. a factorial/fibonacci helper) is ordinary Java and must not recurse forever, and a cycle correctly contributes nothing further rather than falling back to a leaf that gets rejected the same way. Applies uniformly to both the new Java same-class case and the pre-existing Scala nested-closure case, which had the identical one-level limitation. --- .../scala/code/api/util/DynamicUtil.scala | 58 ++++++++++++------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 593f39a7a9..ef26937b6b 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -193,30 +193,48 @@ object DynamicUtil extends MdcLoggable{ val classPool = getClassPool(clazz.getClassLoader) //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. val ctClass = classPool.get(className) + + // A same-class (or same-generated-unit, for the Scala nested-closure case below) call is not + // itself a dependency to police -- recurse into what the TARGET method calls instead of + // flagging the call itself as forbidden, all the way down until a genuinely foreign + // dependency is reached. This is required for Java: every Java dynamic resource doc + // implements Supplier> (the documented convention), and the + // compiler always erases that generic Supplier.get() to a synthetic bridge method + // `Object get()` whose body is just `return this.get();` -- an ordinary same-class + // invokevirtual call to the real, properly-typed get(). A single level of unrolling only + // fixes that one hop: any Java body that factors logic into its own private helper methods + // (an entirely normal thing to do) reintroduces the exact same false rejection one level + // deeper, since the un-recursed helper's own callees would otherwise be appended as raw + // (thisClass, method) tuples and then rejected as calls to an unwhitelistable random-UUID + // class. `visited` guards against a call cycle -- direct or mutual recursion between + // same-class private methods (e.g. a fibonacci/factorial helper) is entirely normal Java and + // would otherwise recurse forever. On hitting a cycle this contributes nothing further (Nil), + // not a leaf: the recursive call is still a same-class call, not a foreign dependency, and + // whatever it in turn depends on is already being expanded by the in-progress call further up + // this same path -- returning it as a leaf here would flag the method's own name + // (unwhitelistable, like any other randomly-named dynamic class) as a forbidden dependency, + // exactly the bug this whole function exists to avoid. + def expand(typeName: String, methodName: String, signature: String, visited: Set[(String, String, String)]): List[(String, String, String)] = { + val key = (typeName, methodName, signature) + val sameUnit = typeName == className || + (className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName + "$")) + if (!sameUnit) { + List(key) + } else if (visited.contains(key)) { + Nil + } else { + APIUtil.getDependentMethods(typeName, methodName, signature, force).flatMap { case (t, m, s) => + expand(t, m, s, visited + key) + } + } + } + for { method <- ctClass.getDeclaredMethods.toList if predicate(method.getName) - ternary @ (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature, force) + (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature, force) } yield { - // if method is also dynamic compile code, extract it's dependent method - if (typeName == className) { - // A same-class self-call is not itself a dependency to police -- recurse into what the - // target method calls instead of flagging the call as forbidden. This is required for - // Java: every Java dynamic resource doc implements Supplier> - // (the documented convention), and the compiler always erases that generic Supplier.get() - // to a synthetic bridge method `Object get()` whose body is just `return this.get();` -- - // an ordinary same-class invokevirtual call to the real, properly-typed get(). Without this - // branch that bridge call is scanned like any other, resolves to (thisClass, "get", ...), - // and since the dynamically-compiled class lives under the OBP-owned code.* package - // (isRestrictedType) but its randomly-generated name can never appear in a static - // whitelist, EVERY Java doc is unconditionally rejected under strict validation -- - // regardless of what its code actually does. - listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature, force)) - } else if(className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName+ "$")) { - listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature, force)) - } else { - listBuffer.append(ternary) - } + listBuffer.appendAll(expand(typeName, methodName, signature, Set.empty)) } listBuffer.distinct.toList From b745291ce316f6a1fc28f7f86456b36bf1d5b225 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 16:40:52 +0200 Subject: [PATCH 15/20] fix: reject unsupported programming_lang in the v6 validate endpoint POST /obp/v6.0.0/management/dynamic-resource-docs/validate re-implemented the request_verb and example_request_body preconditions the v4 create endpoint checks, but not the programming_lang check added alongside Java support. CompiledObjects falls through to the Scala compile path for any value it doesn't recognise as "java"/"Java", so validate would report valid=true for a body that happens to compile as Scala under a bogus or misspelled programming_lang -- a verdict that contradicts what the create endpoint would actually do (400 DynamicCodeLangNotSupport), defeating the point of a dry-run validation endpoint. Add the same language check, mirroring Http4s400's validateDynamicResourceDocBody. No prior test exercised this endpoint at all; added coverage for the rejection and for both supported languages. --- .../scala/code/api/v6_0_0/Http4s600.scala | 11 +++ .../ValidateDynamicResourceDocTest.scala | 81 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index bc5559d9b6..d3a2d4dc6d 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -4427,6 +4427,17 @@ object Http4s600 { case _ => true } } + // Mirrors Http4s400's validateDynamicResourceDocBody: fail fast on an unsupported + // programming_lang here too, rather than reporting `valid = true` for a language + // create would actually reject with 400 DynamicCodeLangNotSupport (CompiledObjects + // silently falls through to the Scala compile path for any value it doesn't + // recognise as Java, so an unsupported/misspelled language would otherwise still + // "validate" successfully as Scala). + _ <- Helper.booleanToFuture( + s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: Scala, Java""", + cc = Some(cc)) { + Set("", "scala", "Scala", "java", "Java").contains(body.programmingLang) + } } yield try { code.api.dynamic.endpoint.helper.CompiledObjects( body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency() diff --git a/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala new file mode 100644 index 0000000000..7637619c6b --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala @@ -0,0 +1,81 @@ +package code.api.v6_0_0 + +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.entitlement.Entitlement +import org.json4s.native.Serialization.write + +/** + * `POST /obp/v6.0.0/management/dynamic-resource-docs/validate` must reject an unsupported + * `programming_lang` the same way `POST .../dynamic-resource-docs` (create) does, rather than + * reporting `valid = true` for a language create would actually 400 on -- see + * Http4s600.validateDynamicResourceDoc's own doc comment: CompiledObjects falls through to the + * Scala compile path for any programming_lang value it doesn't recognise as Java, so a body that + * happens to be valid Scala would otherwise "validate" successfully under a bogus/misspelled + * language. + */ +class ValidateDynamicResourceDocTest extends V600ServerSetup { + + private def validateRequest = (v6_0_0_Request / "management" / "dynamic-resource-docs" / "validate").POST <@ (user1) + + private def docWith(programmingLang: String) = + SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + programmingLang = programmingLang + ) + + feature("Validate Dynamic Resource Doc rejects an unsupported programming_lang") { + scenario("An unsupported programming_lang is rejected, not silently validated as Scala") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val resp = makePostRequest(validateRequest, write(docWith("Python"))) + + Then("the request is rejected with 400, not a 200 valid=true/false body") + resp.code should equal(400) + resp.body.toString should include("OBP-40049") + } + + scenario("programming_lang \"Scala\" is accepted (baseline, unaffected by the language check)") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val resp = makePostRequest(validateRequest, write(docWith("Scala"))) + + Then("the request reaches the compile step and responds 200") + resp.code should equal(200) + (resp.body \ "valid").values should equal(true) + } + + scenario("programming_lang \"Java\" is accepted (baseline, unaffected by the language check)") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val javaBody = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class ValidateEndpointJavaProbe implements Supplier> { + | private Object apply(Object[] args) { + | Map response = new LinkedHashMap<>(); + | response.put("greeting", "hello"); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + val doc = docWith("Java").copy(methodBody = java.net.URLEncoder.encode(javaBody, "UTF-8")) + val resp = makePostRequest(validateRequest, write(doc)) + + Then("the request reaches the compile step and responds 200") + resp.code should equal(200) + (resp.body \ "valid").values should equal(true) + } + } +} From 0fbd2f5279012d6a262735c80cbb0bf0e8cff515 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 16:42:23 +0200 Subject: [PATCH 16/20] fix: log a whitelist rejection distinctly from a genuine compile failure resourceDocs' per-row catch swallows any exception from toResourceDoc and logs it as "likely stored under the deprecated Lift contract -- re-author the body against the new native contract", regardless of cause. That is accurate for a genuine compile failure, but CompiledObjects.validateDependency now runs fresh on every construction (not just at create/update time), so a previously-registered doc can start throwing JsonResponseException here too -- purely because dynamic_code_compile_validate_dependencies was tightened after it was registered. The endpoint silently drops from the listing either way, but the log sends whoever reads it toward rewriting a body that was never the problem. Catch JsonResponseException separately and report the actual whitelist rejection reason (via the same JsonResponseExtractor pattern Http4s600's validate endpoint already uses for this exception shape), falling back to the existing generic message for every other cause. No dedicated test: this only changes log wording for a rejection path already covered end-to-end by DynamicResourceDocJavaSecurityValidationTest; asserting on log content here would need capture infrastructure this codebase doesn't otherwise use for a diagnostics-only change. --- .../DynamicResourceDocsEndpointGroup.scala | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala index 71b33776fd..a20ce39ea1 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala @@ -21,6 +21,22 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel try { Some(toResourceDoc(dynamicDoc)) } catch { + // Validation.validateDependency / createJavaHttp4sEndpoint's own rejection path both throw + // this specifically for a dependency-whitelist miss -- distinct from a genuine compile + // failure, and reachable here (not just at create/update time) because CompiledObjects' + // validation runs fresh on every construction and dynamic_code_compile_validate_dependencies + // can be tightened after a doc was already registered. Logging it as a "deprecated Lift + // contract" problem sends whoever reads this log to re-author a body that is not the + // problem, instead of at the whitelist they (or someone else) just edited. + case e: code.api.JsonResponseException => + val reason = e.jsonResponse match { + case APIUtil.JsonResponseExtractor(msg, _) => msg + case _ => Option(e.getMessage).getOrElse("") + } + logger.error(s"[DynamicResourceDocsEndpointGroup] skipping dynamic resource doc '${dynamicDoc.requestVerb} ${dynamicDoc.requestUrl}' " + + s"(id=${dynamicDoc.dynamicResourceDocId.getOrElse("")}, programming_lang=${dynamicDoc.programmingLang}): rejected by dependency " + + s"validation (dynamic_code_compile_validate_dependencies). $reason") + None case e: Throwable => logger.error(s"[DynamicResourceDocsEndpointGroup] skipping dynamic resource doc '${dynamicDoc.requestVerb} ${dynamicDoc.requestUrl}' " + s"(id=${dynamicDoc.dynamicResourceDocId.getOrElse("")}): its methodBody could not be compiled under the native http4s contract. " + From 7b6057f930171f2b8205103400d8d9c743f923ce Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 16:44:13 +0200 Subject: [PATCH 17/20] perf: bind allowedCompilationMethods once per validateDependency call allowedCompilationMethods was changed from val to def so it observes a live props override (needed for setPropsValues-driven tests), but validateDependency's collect guard reads it twice per dependency tuple -- Map.get and a separate .exists scan. Each read re-derives the whitelist from scratch: a prop read, a regex-based rewrite of the whitelist source into compilable Scala, and a ConcurrentHashMap lookup keyed on that ~1.5KB string. For N dependency tuples that is up to 2N re-derivations where one would do. Bind it to a val at the top of the method instead. Still observes props changes between calls (each validateDependency invocation re-binds), just not multiple times within the same call. --- obp-api/src/main/scala/code/api/util/DynamicUtil.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index ef26937b6b..694cf56145 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -515,6 +515,11 @@ object DynamicUtil extends MdcLoggable{ * Here only validate the restricted types(isObpClass + val restrictedTypes), not all scala/java types. */ private def validateDependency(dependentMethods: List[(String, String, String)]) = { + // Bound once per call, not re-derived per dependency tuple: allowedCompilationMethods is a + // def (see the "def, not val" comment above) so it observes a live props change, but it + // recompiles the whitelist source on every access -- reading it twice per element inside + // the `collect` guard below would mean up to 2N re-derivations for N dependency tuples. + val allowedCompilationMethods = this.allowedCompilationMethods val notAllowedDependentMethods = dependentMethods collect { case (typeName, method, _) if isRestrictedType(typeName) && From 6bb5223eee95ff18392299fd5878ac889e710970 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Tue, 1 Sep 2026 16:45:28 +0200 Subject: [PATCH 18/20] fix: fail fast when a compiled Java class's bytecode is missing createJavaHttp4sEndpoint reflectively reads the compiled class's bytes out of MemoryClassLoader's private mapClassBytes map to hand them to Javassist for dependency validation. The lookup result was passed straight to ByteArrayClassPath unchecked: if a future java-scriptengine version keys that map differently (internal name vs binary name, or drops the field entirely), get() returns null, and the failure only surfaces later as an opaque NPE inside Javassist -- nothing at the actual point of breakage indicates the cause was this reflective coupling to an internal field. Check for null and throw immediately with a message that names the field and the class it was looked up for. The throw happens inside the same Box tryo the rest of the compile step already runs in, so it is reported the same way any other compile failure is -- no behavior change on the success path. Also documents (no functional change) the pre-existing, unbounded-by-design memoClassPool/dynamicCompileResult caching tradeoff that the Java compile path now shares: each distinct compiled method_body retains its ClassLoader and ClassPool for the life of the process. This is consistent with the existing Scala compile cache's design (dynamic resource doc creation is gated behind operator-only roles, not open to arbitrary callers) rather than a new gap introduced here. --- .../scala/code/api/util/DynamicUtil.scala | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 694cf56145..c38de119c4 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -43,6 +43,16 @@ object DynamicUtil extends MdcLoggable{ } val toolBox: ToolBox[universe.type] = runtimeMirror(getClass.getClassLoader).mkToolBox() + // Neither this nor memoJavaCompiledScript below ever evicts, so each distinct ClassLoader (and + // therefore each distinct compiled Java method_body -- java-scriptengine hands createJavaHttp4sEndpoint + // a fresh MemoryClassLoader per compile) is retained for the life of the process, along with its + // ClassPool. This is the same unbounded-but-trusted-operator-only tradeoff dynamicCompileResult + // below already makes for the Scala compile cache, predating the Java path: registering a dynamic + // resource doc is gated behind canCreateDynamicResourceDoc / canCreateBankLevelDynamicResourceDoc, + // not open to arbitrary callers, and a served endpoint's ClassLoader must stay reachable for as + // long as that endpoint keeps serving requests -- an eviction policy here would need to be + // reference-counted against currently-registered docs to avoid reclaiming a live one, which is a + // larger change than this cache's existing (pre-Java) design accounted for. private val memoClassPool = new Memo[ClassLoader, ClassPool] // Caches only the compiled artifact (deterministic given the source string), never the // validation outcome built on top of it -- see createJavaHttp4sEndpoint's doc comment. @@ -753,6 +763,16 @@ object DynamicUtil extends MdcLoggable{ classBytesField.setAccessible(true) val classBytes = classBytesField.get(compiledClass.getClassLoader) .asInstanceOf[java.util.Map[String, Array[Byte]]].get(compiledClass.getName) + // Fail loudly and specifically here rather than handing Javassist a null byte array -- + // that would only surface later, inside ByteArrayClassPath/ClassPool, as an opaque NPE + // with no indication that the cause was this reflective read (e.g. a java-scriptengine + // upgrade that changes mapClassBytes' keying from binary name to internal name, or that + // stops using that field name at all). + if (classBytes == null) { + throw new IllegalStateException( + s"createJavaHttp4sEndpoint: MemoryClassLoader.mapClassBytes has no entry for " + + s"${compiledClass.getName} -- java-scriptengine's internal layout may have changed") + } getClassPool(compiledClass.getClassLoader) .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes)) From a48f7d9b46eb77c2b3815af5c584675748b01b59 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Thu, 1 Oct 2026 09:02:55 +0200 Subject: [PATCH 19/20] Update OPEN_CORRIDOR_INTERFACE_C_PUBLISH_PLAN.md --- OPEN_CORRIDOR_INTERFACE_C_PUBLISH_PLAN.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/OPEN_CORRIDOR_INTERFACE_C_PUBLISH_PLAN.md b/OPEN_CORRIDOR_INTERFACE_C_PUBLISH_PLAN.md index 7242e077bc..b24839e154 100644 --- a/OPEN_CORRIDOR_INTERFACE_C_PUBLISH_PLAN.md +++ b/OPEN_CORRIDOR_INTERFACE_C_PUBLISH_PLAN.md @@ -356,10 +356,11 @@ parts: > (`OBP-OUTGOING-SETTLEMENT-ACCOUNT` → `OBP-INCOMING-SETTLEMENT-ACCOUNT`), > writes `settled_by_transaction_ids` AND `settled_by_transaction_request_id` > (TR B's id — kept even at net zero) attributes on each covered promise, -> flips them COMPLETED, and enqueues the messages into the `OpenCorridorOutbox` -> Mapper table in the same request DB transaction. `OpenCorridorOutboxRelay` -> (Boot-started when `open_corridor_enabled`, `open_corridor.outbox_relay_interval` -> default 10s) publishes with exponential backoff and records each §4.2 reply: +> flips them COMPLETED, and enqueues the messages into the `MessageOutbox` +> Mapper table in the same request DB transaction. `MessageOutboxRelay` +> (Boot-started on every instance, relaying Open Corridor rows only when +> `open_corridor_enabled`; interval `message_outbox.relay_interval_seconds`, +> default 10s, formerly `open_corridor.outbox_relay_interval`) publishes with exponential backoff and records each §4.2 reply: > settlement rows stay PENDING through SUBMITTED/SETTLING (redelivery-as-polling, > §4.4) until FINAL; refutable business errors (COMMITMENT-MISMATCH etc.) go > STICKY for operator reconciliation, never swallowed. Fails fast pre-mutation From 6e3fbdffb51ed8bcf94814e5e282c00e6df6c7fa Mon Sep 17 00:00:00 2001 From: simonredfern Date: Thu, 1 Oct 2026 09:05:58 +0200 Subject: [PATCH 20/20] feat: bind dynamic code approvals to the programming language; programming_lang on v7.0.0 dry-run compile Code hashes cover the programming language as well as the method body. The maker/checker guard recomputes the hash from the row instead of trusting MethodBodyHash. Existing rows are rehashed once at boot, carrying over only approvals still valid for the current body. Approving a queued Java Dynamic Resource Doc now compiles it as Java. POST /management/dynamic-resource-docs/compile takes programming_lang and checks Java bodies without loading them. The Scala dry run checks code inside a method, as the real compile does, so a body using return is no longer rejected. --- docs/MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md | 17 +- .../main/scala/bootstrap/liftweb/Boot.scala | 4 + .../endpoint/helper/DynamicEndpoints.scala | 37 +++- .../main/scala/code/api/util/APIUtil.scala | 23 +++ .../scala/code/api/util/DynamicUtil.scala | 63 ++++++- .../main/scala/code/api/util/Glossary.scala | 2 +- .../scala/code/api/v4_0_0/Http4s400.scala | 4 +- .../scala/code/api/v6_0_0/Http4s600.scala | 4 +- .../scala/code/api/v7_0_0/Http4s700.scala | 28 ++- .../code/api/v7_0_0/JSONFactory7.0.0.scala | 9 +- .../MappedConnectorMethodProvider.scala | 4 +- .../MappedDynamicMessageDocProvider.scala | 4 +- .../MappedDynamicResourceDocProvider.scala | 4 +- .../dynamicchangerequest/MakerChecker.scala | 123 ++++++++++++-- .../code/api/v4_0_0/ConnectorMethodTest.scala | 2 +- .../api/v4_0_0/DynamicMessageDocTest.scala | 2 +- .../api/v4_0_0/DynamicResourceDocTest.scala | 4 +- .../CompileDynamicResourceDocTest.scala | 160 ++++++++++++++++++ .../api/v7_0_0/DynamicChangeRequestTest.scala | 113 ++++++++++++- .../code/api/v7_0_0/Http4s700RoutesTest.scala | 6 +- 20 files changed, 547 insertions(+), 66 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/v7_0_0/CompileDynamicResourceDocTest.scala diff --git a/docs/MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md b/docs/MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md index e208b367bd..374ae97098 100644 --- a/docs/MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md +++ b/docs/MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md @@ -125,8 +125,11 @@ request. Rows are never deleted; the table is the audit log. - `ApprovedHash` on DynamicResourceDoc, DynamicMessageDoc, ConnectorMethod, AbacRule. - `IsActive` (default `true`) on every runtime-loaded dynamic model that lacks it. -The runtime loads a row only if `IsActive` is true and, for code, only if `MethodBodyHash == -ApprovedHash`. When maker/checker is disabled for a target type the hash check is skipped. +The runtime loads a row only if `IsActive` is true and, for code, only if the row's code hash +equals `ApprovedHash`. The code hash is recomputed from the row (its programming language and its +decoded method body, `APIUtil.dynamicCodeHash`) rather than read from the stored `MethodBodyHash`, so +an edit made directly in the database, to the body or to the language, withdraws the approval. When +maker/checker is disabled for a target type the hash check is skipped. ## 4. Endpoints (v7.0.0) @@ -193,8 +196,14 @@ fail loudly. On approval the server, in order: Rejection and withdrawal take `{ "comment": "…" }`. Withdrawal is by the requestor only. *(impl)* Hashes are bare SHA-256 hex, matching the existing `MethodBodyHash` column; a `sha256:` prefix is accepted on approval. `payload_hash` is over the canonical JSON of the request body; -`current_payload_hash` and `ApprovedHash` are the target's decoded method body hash (for ABAC -rules, the hash of `rule_code`). +`current_payload_hash` and `ApprovedHash` are the target's code hash: SHA-256 of its programming +language (trimmed, lower case, blank meaning `scala`), a newline, and its decoded method body (for +ABAC rules, the hash of `rule_code`). The language is included because the same text can be stored +as Scala, Java or Javascript and is compiled by the language's compiler, so approving a body as one +language must not approve it as another. The language version is not included: it belongs to the +deployed runtime, and including it would withdraw every approval at each compiler upgrade. Until +2026-10 the hash covered the body only; `MakerChecker.rehashDynamicCodeWithLanguage` moves existing +rows once at boot, carrying over only the approvals still valid for the row's current body. ### Reading diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index 869a7a1b9c..070bc35719 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -315,6 +315,10 @@ class Boot extends MdcLoggable { // Please note that migration scripts are executed after Lift Mapper Schemifier Migration.database.executeScripts(startedBeforeSchemifier = false) + // Code hashes cover the programming language as well as the body; move existing rows (and the + // approvals still valid for them) to that form once. Must run before the seed below. + code.dynamicchangerequest.MakerChecker.rehashDynamicCodeWithLanguage() + // Maker/checker for dynamic code: when first enabled, pre-existing code rows get their current // body hash recorded as approved so enabling the feature does not silently disable them. code.dynamicchangerequest.MakerChecker.seedApprovedHashesIfEnabled() diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala index 54333b119f..0df1554632 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala @@ -33,7 +33,7 @@ import code.api.dynamic.endpoint.helper.practise.{DynamicEndpointCodeGenerator, import code.api.dynamic.endpoint.helper.practise.PractiseEndpointGroup import code.api.util.DynamicUtil.{DynamicCodeBody, Validation} import code.api.util.APIUtil.{BooleanBody, DoubleBody, EmptyBody, LongBody, Http4sEndpointIO, PrimaryDataBody, ResourceDoc, StringBody, getDisabledEndpointOperationIds} -import code.api.util.{CallContext, DynamicUtil} +import code.api.util.{APIUtil, CallContext, DynamicUtil} import net.liftweb.common.{Box, Failure, Full} import org.json4s.{JNothing, JValue} import org.json4s.JsonAST.{JBool, JDouble, JInt, JString} @@ -105,6 +105,17 @@ trait EndpointGroup { * @param methodBody it is url-encoded string for the api level code. */ object CompiledObjects { + /** + * This is the set of `programming_lang` values a Dynamic Resource Doc may have, compared after + * APIUtil.normaliseDynamicCodeLanguage (trimmed, lower case, blank meaning Scala). Create, update, + * validate and the dry-run compile all check against it, and CompiledObjects chooses its compiler + * from the same normalised value, so nothing that passes the check can reach the wrong compiler. + */ + val supportedLanguages: List[String] = List("scala", "java") + def isSupportedLanguage(programmingLang: String): Boolean = + supportedLanguages.contains(APIUtil.normaliseDynamicCodeLanguage(programmingLang)) + val supportedLanguagesText: String = "Scala, Java" + /** * The native http4s template a method body is inlined into. Returns the full source and the * 1-based line on which the method body starts, so compiler positions can be mapped back to it. @@ -153,10 +164,18 @@ object CompiledObjects { /** * Dry run without constructing a CompiledObjects (whose constructor compiles for real): compiler - * diagnostics with line numbers relative to the method body the author wrote. Empty = compiles. + * diagnostics with line numbers relative to the method body the author wrote, in the body's + * programming language (Scala or Java). Empty = compiles. */ - def compileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String): List[DynamicUtil.CompileProblem] = { + def compileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String, programmingLang: String = "Scala"): List[DynamicUtil.CompileProblem] = { val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8") + // Java bodies are compiled as written (no template, no generated case classes), so the example + // bodies play no part, as they play none in CompiledObjects' own Java branch. + if (APIUtil.normaliseDynamicCodeLanguage(programmingLang) == "java") DynamicUtil.checkJavaCode(decodedMethodBody) + else scalaCompileProblems(exampleRequestBody, successResponseBody, decodedMethodBody) + } + + private def scalaCompileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], decodedMethodBody: String): List[DynamicUtil.CompileProblem] = { val requestBody: Product = exampleRequestBody match { case Some(JString(s)) if StringUtils.isBlank(s) => toCaseObject(None) case _ => toCaseObject(exampleRequestBody) @@ -196,15 +215,15 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo } val successResponse: Product = toCaseObject(successResponseBody) - private val partialFunction: Http4sEndpointIO = programmingLang match { - case "java" | "Java" => + private val partialFunction: Http4sEndpointIO = APIUtil.normaliseDynamicCodeLanguage(programmingLang) match { + case "java" => DynamicUtil.createJavaHttp4sEndpoint(decodedMethodBody) match { case Full(func) => func case Failure(msg: String, exception: Box[Throwable], _) => throw exception.getOrElse(new RuntimeException(msg)) case _ => throw new RuntimeException("compiled code return nothing") } - case _ /* "Scala" | "scala" | "" | null, default */ => + case _ /* "scala", the default; create and update reject anything else (isSupportedLanguage) */ => scalaPartialFunction } @@ -265,8 +284,8 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo * internally (see its own doc comment) before ever returning that wrapper, so re-validating the * wrapper here is both redundant and wrong -- it would reject every Java doc unconditionally. */ - def validateDependency() = programmingLang match { - case "java" | "Java" => () + def validateDependency() = APIUtil.normaliseDynamicCodeLanguage(programmingLang) match { + case "java" => () case _ => Validation.validateDependency(this.partialFunction) } @@ -275,7 +294,7 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo * author wrote (the wrapper's own lines are subtracted). Empty = compiles. Nothing is evaluated or cached. */ def compileProblems(): List[DynamicUtil.CompileProblem] = - CompiledObjects.compileProblems(exampleRequestBody, successResponseBody, methodBody) + CompiledObjects.compileProblems(exampleRequestBody, successResponseBody, methodBody, programmingLang) /** * Wraps the compiled partial function as an endpoint. This used to bind a per-bank diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 9a7aa7f09b..a281069569 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -3410,6 +3410,29 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ digest.map(b => f"$b%02x").mkString } + /** + * This is the hash stored as `MethodBodyHash` on runtime-compiled code (Dynamic Resource Docs, + * Dynamic Message Docs, Connector Methods), and therefore the value a maker/checker approval binds + * to (`ApprovedHash`). + * + * It covers the programming language as well as the body. The same text can be stored as Scala, + * Java or Javascript, and the runtime picks a compiler from the language, so an approval of a body + * as one language must not carry over when only the language is changed. The language is + * normalised (trimmed, lower case, blank meaning `scala`, the default every one of these types + * applies), because the runtime treats `Java` and `java` as the same language. + * + * The language version is deliberately not included: it is a property of the deployed runtime, not + * of the row, and including it would withdraw every approval at each compiler upgrade. + * + * Until 2026-10 the hash covered the body only; `MakerChecker.rehashDynamicCodeWithLanguage` moves + * existing rows to this form once. + */ + def dynamicCodeHash(programmingLang: String, decodedMethodBody: String): String = + sha256Hex(normaliseDynamicCodeLanguage(programmingLang) + "\n" + Option(decodedMethodBody).getOrElse("")) + + def normaliseDynamicCodeLanguage(programmingLang: String): String = + Option(programmingLang).map(_.trim.toLowerCase).filter(_.nonEmpty).getOrElse("scala") + /** * Create the explicit CounterpartyId, (Used in `Create counterparty for an account` endpoint ). * This is just a UUID, use both in Counterparty.counterpartyId and CounterpartyMetadata.counterpartyId diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index 8b3033ed17..b7a7ec2694 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -84,22 +84,81 @@ object DynamicUtil extends MdcLoggable{ /** * Compile `code` for diagnostics only: nothing is evaluated, nothing is cached. Empty list = compiles. * Toolboxes are not thread-safe, so checks are serialised. Callers must apply the kill switch and a role. + * + * The code is checked inside a method, because that is how it is really compiled: ToolBox.compile + * wraps the tree in a method before compiling it. Checked bare, a `return` in the code (which the + * documented example method body uses to answer early with an error) is reported as "return + * outside method definition", although a real create accepts it. The wrapper is added on the same + * line as the start of the code, so line numbers are unchanged; columns on that first line are + * corrected for it. */ def checkScalaCode(code: String): List[CompileProblem] = checkToolBox.synchronized { checkFrontEnd.reset() + val wrapperStart = "{ def dryRunCompileWrapper(): Any = { " + val wrapperEnd = "\n}; () }" val failure: Option[String] = try { - checkToolBox.typecheck(checkToolBox.parse(code)) + checkToolBox.typecheck(checkToolBox.parse(wrapperStart + code + wrapperEnd)) None } catch { case e: ToolBoxError => Some(e.message) } val collected = checkFrontEnd.infos.toList.filter(_.severity == checkFrontEnd.ERROR).map { info => val (line, column) = if (info.pos != null && info.pos.isDefined) (info.pos.line, info.pos.column) else (0, 0) - CompileProblem(line, column, "ERROR", info.msg) + val columnInCode = if (line == 1 && column > wrapperStart.length) column - wrapperStart.length else column + CompileProblem(line, columnInCode, "ERROR", info.msg) } if (collected.nonEmpty) collected else failure.map(m => CompileProblem(0, 0, "ERROR", m.stripPrefix("reflective typecheck has failed:").stripPrefix("reflective compilation has failed:").trim)).toList } + + /** + * This compiles a Java `method_body` for diagnostics only, the Java counterpart of + * [[checkScalaCode]]. Empty list = compiles. Line numbers are relative to the body the author + * wrote. + * + * The source is prepared exactly as createJavaHttp4sEndpoint prepares it (the author's `package` + * line replaced by a fresh generated package), and compiled by the same system Java compiler with + * the same in-memory file manager, so it reports what a real create would report. Unlike the real + * compile, nothing is loaded or constructed: java-scriptengine's own compile also instantiates the + * class, which would run the author's constructor and static initialisers. Nothing is cached. + * Callers must apply the kill switch and a role. + */ + def checkJavaCode(methodBody: String): List[CompileProblem] = { + import javax.tools.{Diagnostic, DiagnosticCollector, JavaFileObject, ToolProvider} + import scala.jdk.CollectionConverters._ + + val compiler = ToolProvider.getSystemJavaCompiler + if (compiler == null) { + List(CompileProblem(0, 0, "ERROR", "No Java compiler is available: this server runs on a Java runtime without the compiler (a JRE rather than a JDK).")) + } else { + // One line is added above the author's code: the generated package statement. + val linesAdded = 1 + val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) + val packageName = "code.api.util.dynamic." + UUID.randomUUID().toString.replaceAll("^|-", "_") + val javaCode = s"package $packageName;\n${packageMatcher.replaceFirst("")}\n" + + (Box tryo { new ch.obermuhlner.scriptengine.java.name.DefaultNameStrategy().getFullName(javaCode) }) match { + case Full(fullClassName) => + val diagnostics = new DiagnosticCollector[JavaFileObject] + val fileManager = new ch.obermuhlner.scriptengine.java.MemoryFileManager( + compiler.getStandardFileManager(diagnostics, null, null), null) + val simpleClassName = StringUtils.substringAfterLast("." + fullClassName, ".") + val source = fileManager.createSourceFileObject(null, simpleClassName, javaCode) + val compiled: Boolean = compiler.getTask(null, fileManager, diagnostics, null, null, java.util.Arrays.asList(source)).call() + val problems = diagnostics.getDiagnostics.asScala.toList.filter(_.getKind == Diagnostic.Kind.ERROR).map { d => + val line = if (d.getLineNumber > linesAdded) (d.getLineNumber - linesAdded).toInt else 0 + val column = if (line > 0 && d.getColumnNumber > 0) d.getColumnNumber.toInt else 0 + CompileProblem(line, column, "ERROR", d.getMessage(java.util.Locale.ENGLISH)) + } + if (problems.nonEmpty || compiled) problems + else List(CompileProblem(0, 0, "ERROR", "The Java compiler rejected the method body without reporting why.")) + case failure => + // DefaultNameStrategy fails when there is no class declaration to name the source file after. + val reason = failure match { case f: Failure => f.msg; case _ => "no class declaration found" } + List(CompileProblem(0, 0, "ERROR", s"A Java method body must declare a public class implementing Supplier>: $reason")) + } + } + } // Neither this nor memoJavaCompiledScript below ever evicts, so each distinct ClassLoader (and // therefore each distinct compiled Java method_body -- java-scriptengine hands createJavaHttp4sEndpoint // a fresh MemoryClassLoader per compile) is retained for the life of the process, along with its diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 09c03a6db4..9a2bd7f4ab 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -4242,7 +4242,7 @@ object Glossary extends MdcLoggable { | |2) The checker reads the request (`GET /obp/v7.0.0/management/dynamic-change-requests/CHANGE_REQUEST_ID`, which returns the proposed and the current payload side by side) and approves it by quoting its `payload_hash`, the SHA-256 of the exact body, on `POST .../approval`. Only then is the change applied. OBP refuses an approval from the User who made the request (`OBP-30279`). | -|3) Content is approved, not records. Any later edit produces a new hash and needs a new approval. The runtime compiles and serves only rows whose body hash equals the hash a checker approved, so a row edited directly in the database does not run. +|3) Content is approved, not records. Any later edit produces a new hash and needs a new approval. The runtime compiles and serves only rows whose code hash equals the hash a checker approved. The code hash covers the programming language as well as the method body, and is recomputed from the row each time, so a row whose body or language is edited directly in the database does not run. | |4) Deactivating an artefact is a direct action by a single checker (`POST .../deactivation`), with no request: four eyes to enable, one pair to disable. Enabling it again goes through a request. | diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index 4bde0237f6..961ef238cf 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -9582,9 +9582,9 @@ object Http4s400 { // Fail fast with a clean 400 before attempting compilation, rather than surfacing an // unsupported programming_lang only as a generic DynamicCodeCompileFail. _ <- code.util.Helper.booleanToFuture( - s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: Scala, Java""", + s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: ${code.api.dynamic.endpoint.helper.CompiledObjects.supportedLanguagesText}""", cc = Some(cc)) { - Set("", "scala", "Scala", "java", "Java").contains(body.programmingLang) + code.api.dynamic.endpoint.helper.CompiledObjects.isSupportedLanguage(body.programmingLang) } } yield () } diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 1b87697a4d..9c12faed85 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -4670,9 +4670,9 @@ object Http4s600 { // recognise as Java, so an unsupported/misspelled language would otherwise still // "validate" successfully as Scala). _ <- Helper.booleanToFuture( - s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: Scala, Java""", + s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: ${code.api.dynamic.endpoint.helper.CompiledObjects.supportedLanguagesText}""", cc = Some(cc)) { - Set("", "scala", "Scala", "java", "Java").contains(body.programmingLang) + code.api.dynamic.endpoint.helper.CompiledObjects.isSupportedLanguage(body.programmingLang) } } yield try { code.api.dynamic.endpoint.helper.CompiledObjects( diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index a96b026b97..1d07bac1fc 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -5893,7 +5893,7 @@ object Http4s700 { "GET", "/management/dynamic-resource-docs", "Get Dynamic Resource Docs (with provenance)", - s"""Returns all Dynamic Resource Docs, each wrapped with a `provenance` object recording who created / last updated the runtime-compiled code and a SHA-256 of its method body. + s"""Returns all Dynamic Resource Docs, each wrapped with a `provenance` object recording who created / last updated the runtime-compiled code and a SHA-256 of its programming language and method body. | |This is the v7.0.0 read view of the v4.0.0 Dynamic Resource Docs; create / update / delete remain on v4.0.0. | @@ -5951,7 +5951,7 @@ object Http4s700 { "GET", "/management/connector-methods", "Get Connector Methods (with provenance)", - s"""Returns all Connector Methods, each wrapped with a `provenance` object recording who created / last updated the runtime-compiled code and a SHA-256 of its method body. + s"""Returns all Connector Methods, each wrapped with a `provenance` object recording who created / last updated the runtime-compiled code and a SHA-256 of its programming language and method body. | |This is the v7.0.0 read view of the v4.0.0 Connector Methods; create / update remain on v4.0.0. | @@ -6009,7 +6009,7 @@ object Http4s700 { "GET", "/management/dynamic-message-docs", "Get Dynamic Message Docs (with provenance)", - s"""Returns all Dynamic Message Docs, each wrapped with a `provenance` object recording who created / last updated the runtime-compiled code and a SHA-256 of its method body. + s"""Returns all Dynamic Message Docs, each wrapped with a `provenance` object recording who created / last updated the runtime-compiled code and a SHA-256 of its programming language and method body. | |This is the v7.0.0 read view of the v4.0.0 Dynamic Message Docs; create / update / delete remain on v4.0.0. | @@ -6821,15 +6821,21 @@ object Http4s700 { _ <- code.util.Helper.booleanToFuture(s"""$InvalidJsonFormat The request_verb must be one of ["POST", "PUT", "GET", "DELETE"]""", cc = Some(cc)) { Set("POST", "PUT", "GET", "DELETE").contains(body.request_verb) } + programmingLang = body.programming_lang.getOrElse("Scala") + _ <- code.util.Helper.booleanToFuture( + s"""${code.api.util.ErrorMessages.DynamicCodeLangNotSupport} programming_lang $programmingLang, currently supported languages: ${code.api.dynamic.endpoint.helper.CompiledObjects.supportedLanguagesText}""", + cc = Some(cc)) { + code.api.dynamic.endpoint.helper.CompiledObjects.isSupportedLanguage(programmingLang) + } result <- Future { val start = System.currentTimeMillis() - val problems = scala.util.Try(code.api.dynamic.endpoint.helper.CompiledObjects.compileProblems(body.example_request_body, body.success_response_body, body.method_body)) match { + val problems = scala.util.Try(code.api.dynamic.endpoint.helper.CompiledObjects.compileProblems(body.example_request_body, body.success_response_body, body.method_body, programmingLang)) match { case scala.util.Success(ps) => ps case scala.util.Failure(e) => List(code.api.util.DynamicUtil.CompileProblem(0, 0, "ERROR", Option(e.getMessage).getOrElse(e.toString))) } val dependencyError: Option[String] = if (problems.nonEmpty) None - else scala.util.Try(code.api.dynamic.endpoint.helper.CompiledObjects(body.example_request_body, body.success_response_body, body.method_body).validateDependency()) match { + else scala.util.Try(code.api.dynamic.endpoint.helper.CompiledObjects(body.example_request_body, body.success_response_body, body.method_body, programmingLang).validateDependency()) match { case scala.util.Success(_) => None case scala.util.Failure(e: code.api.JsonResponseException) => Some(com.openbankproject.commons.util.JsonAliases.compactRender(e.jsonResponse.body)) case scala.util.Failure(e) => Some(Option(e.getMessage).getOrElse(e.toString)) @@ -6855,11 +6861,17 @@ object Http4s700 { |Send the fields that shape the compiled code: `request_verb`, `request_url`, the URL-encoded `method_body`, and the optional |`example_request_body` and `success_response_body` (they become the generated `RequestRootJsonClass` / `ResponseRootJsonClass`). | - |`errors` carry the compiler's messages with `line` and `column` relative to the method body you sent (the server's wrapper lines are + |`programming_lang` is the language of the method body, as on Create Dynamic Resource Doc: one of ${code.api.dynamic.endpoint.helper.CompiledObjects.supportedLanguagesText}. + |It is optional and defaults to Scala. Any other value is rejected with ${code.api.util.ErrorMessages.DynamicCodeLangNotSupport.takeWhile(_ != ':')}. + |A Java body is compiled as written, so `example_request_body` and `success_response_body` do not affect it. It must declare a public class + |implementing `Supplier>`; the function receives the raw request body, the path parameters and the CallContext. + | + |`errors` carry the compiler's messages with `line` and `column` relative to the method body you sent (the server's added lines are |subtracted; 0 when the compiler gave no position). When the body compiles and `dynamic_code_obp_calls_are_restricted` is on, |the dependency validator runs too and any forbidden call is reported in `dependency_error`. `compiles` is true only when both pass. | - |Nothing is evaluated or cached, but compiling is a full scalac run, so the same rules apply as for creating: the + |The compiler diagnostics are produced without running anything. When the body compiles, the dependency check builds it as Create would, + |so the same rules apply as for creating: the |`allow_user_generated_scala_code` kill switch, the create role, and at most $dynamicCompileCallsPerMinute calls per minute per user. | |Built for editors that let an author, or an assistant such as Opey, iterate on a body until it compiles before submitting it. @@ -6867,7 +6879,7 @@ object Http4s700 { |${userAuthenticationMessage(true)}""".stripMargin, JSONFactory700.dynamicResourceDocCompileJsonV700Example, JSONFactory700.dynamicCompileResultJsonV700Example, - List($AuthenticatedUserIsRequired, InvalidJsonFormat, UserHasMissingRoles, DynamicCodeExecutionDisabled, code.api.util.ErrorMessages.TooManyRequests, UnknownError), + List($AuthenticatedUserIsRequired, InvalidJsonFormat, UserHasMissingRoles, DynamicCodeExecutionDisabled, code.api.util.ErrorMessages.DynamicCodeLangNotSupport, code.api.util.ErrorMessages.TooManyRequests, UnknownError), apiTagDynamicResourceDoc :: apiTagDynamic :: Nil, Some(List(ApiRole.canCreateDynamicResourceDoc)), http4sPartialFunction = Some(compileDynamicResourceDoc) diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala index 2f8f25d4e6..80ca5fd32e 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala @@ -62,7 +62,7 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { // ─── Provenance for runtime-compiled dynamic code (v7.0.0 read-only exposure) ─── // The v4.0.0 create/update endpoints capture who created / last updated a piece of runtime - // code and a SHA-256 of its (decoded) method body into DB columns, but the v4 response shape + // code and a SHA-256 of its programming language and (decoded) method body into DB columns, but the v4 response shape // is frozen (STABLE) and does not carry them. These v7 GET endpoints expose that provenance, // wrapping the unchanged v4 resource JSON alongside a `provenance` object. case class ProvenanceJsonV700( @@ -1891,7 +1891,9 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { request_url: String, method_body: String, example_request_body: Option[JValue], - success_response_body: Option[JValue] + success_response_body: Option[JValue], + // "Scala" (the default when omitted) or "Java", as on Create Dynamic Resource Doc. + programming_lang: Option[String] = None ) case class DynamicCompileErrorJsonV700(line: Int, column: Int, severity: String, message: String) case class DynamicCompileResultJsonV700( @@ -1905,7 +1907,8 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { request_url = "/hello/world", method_body = java.net.URLEncoder.encode("Future.successful((Map(\"hello\" -> \"world\"), HttpCode.`200`(callContext)))", "UTF-8"), example_request_body = None, - success_response_body = Some(org.json4s.JsonAST.JObject(List(org.json4s.JsonAST.JField("hello", org.json4s.JsonAST.JString("world"))))) + success_response_body = Some(org.json4s.JsonAST.JObject(List(org.json4s.JsonAST.JField("hello", org.json4s.JsonAST.JString("world"))))), + programming_lang = Some("Scala") ) lazy val dynamicCompileResultJsonV700Example = DynamicCompileResultJsonV700( compiles = false, diff --git a/obp-api/src/main/scala/code/connectormethod/MappedConnectorMethodProvider.scala b/obp-api/src/main/scala/code/connectormethod/MappedConnectorMethodProvider.scala index 8bef23be08..6baae5852e 100644 --- a/obp-api/src/main/scala/code/connectormethod/MappedConnectorMethodProvider.scala +++ b/obp-api/src/main/scala/code/connectormethod/MappedConnectorMethodProvider.scala @@ -78,7 +78,7 @@ object MappedConnectorMethodProvider extends ConnectorMethodProvider { .Lang(entity.programmingLang) // provenance is set here from the authenticated user + computed hash, not from `entity` .CreatedByUserId(createdByUserId.getOrElse(null)) - .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) + .MethodBodyHash(APIUtil.dynamicCodeHash(entity.programmingLang, entity.decodedMethodBody)) .saveMe() }.map(ConnectorMethod.getJsonConnectorMethod) @@ -91,7 +91,7 @@ object MappedConnectorMethodProvider extends ConnectorMethodProvider { .Lang(programmingLang) // CreatedByUserId is left untouched; record who last changed the code + refresh the hash .UpdatedByUserId(updatedByUserId.getOrElse(null)) - .MethodBodyHash(APIUtil.sha256Hex(java.net.URLDecoder.decode(connectorMethodBody, "UTF-8"))) + .MethodBodyHash(APIUtil.dynamicCodeHash(programmingLang, java.net.URLDecoder.decode(connectorMethodBody, "UTF-8"))) .saveMe() }.map(ConnectorMethod.getJsonConnectorMethod) case _ => Empty diff --git a/obp-api/src/main/scala/code/dynamicMessageDoc/MappedDynamicMessageDocProvider.scala b/obp-api/src/main/scala/code/dynamicMessageDoc/MappedDynamicMessageDocProvider.scala index b23205999d..4cf20b8dda 100644 --- a/obp-api/src/main/scala/code/dynamicMessageDoc/MappedDynamicMessageDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicMessageDoc/MappedDynamicMessageDocProvider.scala @@ -99,7 +99,7 @@ object MappedDynamicMessageDocProvider extends DynamicMessageDocProvider { .Lang(entity.programmingLang) // provenance is set here from the authenticated user + computed hash, not from `entity` .CreatedByUserId(createdByUserId.getOrElse(null)) - .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) + .MethodBodyHash(APIUtil.dynamicCodeHash(entity.programmingLang, entity.decodedMethodBody)) .saveMe() }.map(DynamicMessageDoc.getJsonDynamicMessageDoc) } @@ -134,7 +134,7 @@ object MappedDynamicMessageDocProvider extends DynamicMessageDocProvider { .Lang(entity.programmingLang) // CreatedByUserId is left untouched; record who last changed the code + refresh the hash .UpdatedByUserId(updatedByUserId.getOrElse(null)) - .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) + .MethodBodyHash(APIUtil.dynamicCodeHash(entity.programmingLang, entity.decodedMethodBody)) .saveMe() }.map(DynamicMessageDoc.getJsonDynamicMessageDoc) case _ => Empty diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala index 1a40e2445a..ac1160efd5 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala @@ -112,7 +112,7 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { .Lang(entity.programmingLang) // provenance is set here from the authenticated user + computed hash, not from `entity` .CreatedByUserId(createdByUserId.getOrElse(null)) - .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) + .MethodBodyHash(APIUtil.dynamicCodeHash(entity.programmingLang, entity.decodedMethodBody)) .saveMe() }.map(DynamicResourceDoc.getJsonDynamicResourceDoc) @@ -138,7 +138,7 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { .Lang(entity.programmingLang) // CreatedByUserId is left untouched; record who last changed the code + refresh the hash .UpdatedByUserId(updatedByUserId.getOrElse(null)) - .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) + .MethodBodyHash(APIUtil.dynamicCodeHash(entity.programmingLang, entity.decodedMethodBody)) .saveMe() }.map(DynamicResourceDoc.getJsonDynamicResourceDoc) case _ => Empty diff --git a/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala b/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala index 0b8a694419..014308916b 100644 --- a/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala +++ b/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala @@ -33,7 +33,7 @@ import java.util.Date import code.abacrule.{AbacRule, AbacRuleEngine, MappedAbacRuleProvider} import code.api.Constant import code.api.dynamic.endpoint.helper.CompiledObjects -import code.api.util.APIUtil.{getPropsAsBoolValue, getPropsAsIntValue, getPropsValue, sha256Hex} +import code.api.util.APIUtil.{dynamicCodeHash, getPropsAsBoolValue, getPropsAsIntValue, getPropsValue, sha256Hex} import code.api.util.DynamicUtil.Validation import code.api.util.{CallContext, ErrorMessages} import code.api.v6_0_0.{CreateAbacRuleJsonV600, UpdateAbacRuleJsonV600} @@ -111,15 +111,21 @@ object MakerChecker extends MdcLoggable { private def blank(s: String): Boolean = StringUtils.isBlank(s) - private def bodyHashOf(storedHash: String, encodedBody: String): String = - if (!blank(storedHash)) storedHash - else sha256Hex(URLDecoder.decode(Option(encodedBody).getOrElse(""), "UTF-8")) + /** + * The hash of a code row as it is now: its language and its decoded body (see APIUtil.dynamicCodeHash). + * + * Always recomputed from the row, never read from the stored MethodBodyHash column. The guard + * compares this with ApprovedHash, and a stored hash would let a direct database edit of the body + * or the language keep the approval of the code it replaced. + */ + private def bodyHashOf(programmingLang: String, encodedBody: String): String = + dynamicCodeHash(programmingLang, URLDecoder.decode(Option(encodedBody).getOrElse(""), "UTF-8")) /** The live target's body hash, Empty when the target does not exist. */ def currentBodyHash(targetType: DynamicChangeRequestTargetType, targetId: String): Box[String] = targetType match { - case DYNAMIC_RESOURCE_DOC => DynamicResourceDoc.find(By(DynamicResourceDoc.DynamicResourceDocId, targetId)).map(r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get)) - case DYNAMIC_MESSAGE_DOC => DynamicMessageDoc.find(By(DynamicMessageDoc.DynamicMessageDocId, targetId)).map(r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get)) - case CONNECTOR_METHOD => ConnectorMethod.find(By(ConnectorMethod.ConnectorMethodId, targetId)).map(r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get)) + case DYNAMIC_RESOURCE_DOC => DynamicResourceDoc.find(By(DynamicResourceDoc.DynamicResourceDocId, targetId)).map(r => bodyHashOf(r.Lang.get, r.MethodBody.get)) + case DYNAMIC_MESSAGE_DOC => DynamicMessageDoc.find(By(DynamicMessageDoc.DynamicMessageDocId, targetId)).map(r => bodyHashOf(r.Lang.get, r.MethodBody.get)) + case CONNECTOR_METHOD => ConnectorMethod.find(By(ConnectorMethod.ConnectorMethodId, targetId)).map(r => bodyHashOf(r.Lang.get, r.MethodBody.get)) case ABAC_RULE => AbacRule.find(By(AbacRule.AbacRuleId, targetId)).map(r => sha256Hex(Option(r.RuleCode.get).getOrElse(""))) case _ => Empty } @@ -138,18 +144,18 @@ object MakerChecker extends MdcLoggable { def isExecutableDynamicResourceDoc(dynamicResourceDocId: String): Boolean = DynamicResourceDoc.find(By(DynamicResourceDoc.DynamicResourceDocId, dynamicResourceDocId)) - .map(r => executable(r.IsActive.get, bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), r.ApprovedHash.get, DYNAMIC_RESOURCE_DOC)) + .map(r => executable(r.IsActive.get, bodyHashOf(r.Lang.get, r.MethodBody.get), r.ApprovedHash.get, DYNAMIC_RESOURCE_DOC)) .getOrElse(false) def isExecutableDynamicMessageDoc(dynamicMessageDocId: String): Boolean = memoGuard("dmd_" + dynamicMessageDocId) { DynamicMessageDoc.find(By(DynamicMessageDoc.DynamicMessageDocId, dynamicMessageDocId)) - .map(r => executable(r.IsActive.get, bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), r.ApprovedHash.get, DYNAMIC_MESSAGE_DOC)) + .map(r => executable(r.IsActive.get, bodyHashOf(r.Lang.get, r.MethodBody.get), r.ApprovedHash.get, DYNAMIC_MESSAGE_DOC)) .getOrElse(false) } def isExecutableConnectorMethod(connectorMethodId: String): Boolean = memoGuard("cm_" + connectorMethodId) { ConnectorMethod.find(By(ConnectorMethod.ConnectorMethodId, connectorMethodId)) - .map(r => executable(r.IsActive.get, bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), r.ApprovedHash.get, CONNECTOR_METHOD)) + .map(r => executable(r.IsActive.get, bodyHashOf(r.Lang.get, r.MethodBody.get), r.ApprovedHash.get, CONNECTOR_METHOD)) .getOrElse(false) } @@ -359,7 +365,7 @@ object MakerChecker extends MdcLoggable { for { body <- parseAs[JsonDynamicResourceDoc](request.proposedPayload) _ <- compileBox("dynamic resource doc") { - val compiled = CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody) + val compiled = CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang) compiled.validateDependency() Full(compiled) } @@ -465,17 +471,17 @@ object MakerChecker extends MdcLoggable { targetType match { case DYNAMIC_RESOURCE_DOC => DynamicResourceDoc.find(By(DynamicResourceDoc.DynamicResourceDocId, targetId)).map { r => - val h = bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get) + val h = bodyHashOf(r.Lang.get, r.MethodBody.get) r.MethodBodyHash(h).ApprovedHash(h).IsActive(true).save; () } case DYNAMIC_MESSAGE_DOC => DynamicMessageDoc.find(By(DynamicMessageDoc.DynamicMessageDocId, targetId)).map { r => - val h = bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get) + val h = bodyHashOf(r.Lang.get, r.MethodBody.get) r.MethodBodyHash(h).ApprovedHash(h).IsActive(true).save; () } case CONNECTOR_METHOD => ConnectorMethod.find(By(ConnectorMethod.ConnectorMethodId, targetId)).map { r => - val h = bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get) + val h = bodyHashOf(r.Lang.get, r.MethodBody.get) r.MethodBodyHash(h).ApprovedHash(h).IsActive(true).save; () } case ABAC_RULE => @@ -500,6 +506,89 @@ object MakerChecker extends MdcLoggable { // ─── boot ────────────────────────────────────────────────────────────────── + /** MigrationScriptLog entry that records the one-off rehash below. */ + val rehashMigrationName = "rehashDynamicCodeWithLanguage" + + /** + * This moves stored code hashes from the old form (SHA-256 of the body alone) to the current form + * (SHA-256 of the language and the body, see APIUtil.dynamicCodeHash). It runs once per database, + * at every boot until it has succeeded, whether or not maker/checker is enabled, because + * MethodBodyHash is also reported as provenance. + * + * It runs from Boot rather than as a Migration.database script because those run only on instances + * that switch migration scripts on. Without this step, every approved row on any other instance + * would stop executing at the upgrade, since its ApprovedHash would no longer equal the hash the + * guard computes. + * + * For each Dynamic Resource Doc, Dynamic Message Doc and Connector Method: + * - MethodBodyHash is recomputed in the new form; + * - ApprovedHash is moved to the new form only when it equals the old-form hash of the row's + * current body, so the approval is carried over exactly where it was still valid. A row whose + * body no longer matches its approval stays unexecutable, as it was before; + * - a pending change request whose CurrentPayloadHash equals the old-form hash of its target is + * moved too, so it does not turn stale merely because of this rehash. + * + * What it cannot do: an approval granted before this change bound the body only, so a row whose + * language was changed after approval (through the API, without a new body) keeps that approval, + * now bound to its current language. From this change on, changing the language requires approval. + * + * Rerunning it is harmless: once moved, an ApprovedHash no longer equals the old form and is left + * alone, so several nodes booting at once cannot do damage. + */ + def rehashDynamicCodeWithLanguage(): Unit = { + val logProvider = code.migration.MigrationScriptLogProvider.migrationScriptLogProvider.vend + if (!logProvider.isExecuted(rehashMigrationName)) { + val start = System.currentTimeMillis() + // targetType -> (old-form hash -> new-form hash) of every row, for moving pending change requests. + val moved = scala.collection.mutable.Map[(String, String), (String, String)]() + def rehash[T](targetType: DynamicChangeRequestTargetType, rows: List[T])( + id: T => String, lang: T => String, encodedBody: T => String, approvedHash: T => String, write: (T, String, Option[String]) => Unit + ): String = { + var approvalsMoved = 0 + rows.foreach { r => + val decoded = URLDecoder.decode(Option(encodedBody(r)).getOrElse(""), "UTF-8") + val oldHash = sha256Hex(decoded) + val newHash = dynamicCodeHash(lang(r), decoded) + val approvalStillValid = approvedHash(r) == oldHash + if (approvalStillValid) approvalsMoved += 1 + write(r, newHash, if (approvalStillValid) Some(newHash) else None) + moved((targetType.toString, id(r))) = (oldHash, newHash) + } + s"$targetType: ${rows.size} rows, $approvalsMoved approvals moved" + } + tryo { + val summary = List( + rehash(DYNAMIC_RESOURCE_DOC, DynamicResourceDoc.findAll())( + _.DynamicResourceDocId.get, _.Lang.get, _.MethodBody.get, _.ApprovedHash.get, + (r, h, approved) => { r.MethodBodyHash(h); approved.foreach(r.ApprovedHash(_)); r.save; () }), + rehash(DYNAMIC_MESSAGE_DOC, DynamicMessageDoc.findAll())( + _.DynamicMessageDocId.get, _.Lang.get, _.MethodBody.get, _.ApprovedHash.get, + (r, h, approved) => { r.MethodBodyHash(h); approved.foreach(r.ApprovedHash(_)); r.save; () }), + rehash(CONNECTOR_METHOD, ConnectorMethod.findAll())( + _.ConnectorMethodId.get, _.Lang.get, _.MethodBody.get, _.ApprovedHash.get, + (r, h, approved) => { r.MethodBodyHash(h); approved.foreach(r.ApprovedHash(_)); r.save; () }) + ) + val pending = DynamicChangeRequest.findAll(By(DynamicChangeRequest.Status, DynamicChangeRequestStatus.INITIATED.toString)) + var requestsMoved = 0 + pending.foreach { request => + moved.get((request.TargetType.get, request.TargetId.get)).foreach { case (oldHash, newHash) => + if (request.CurrentPayloadHash.get == oldHash) { request.CurrentPayloadHash(newHash).save; requestsMoved += 1 } + } + } + (summary :+ s"pending change requests moved: $requestsMoved").mkString(", ") + } match { + case Full(summary) => + val comment = s"Code hashes now cover the programming language as well as the body ($summary)" + logger.warn(s"rehashDynamicCodeWithLanguage says: $comment") + logProvider.saveLog(rehashMigrationName, code.api.util.APIUtil.gitCommit, true, start, System.currentTimeMillis(), comment) + case f: Failure => + logger.error(s"rehashDynamicCodeWithLanguage says: failed, will retry at next boot: ${f.messageChain}") + logProvider.saveLog(rehashMigrationName, code.api.util.APIUtil.gitCommit, false, start, System.currentTimeMillis(), f.messageChain) + case _ => () + } + } + } + /** MigrationScriptLog entry that records the one-off seeding below; the seed never runs twice on a database. */ val seedMigrationName = "seedDynamicCodeApprovedHashes" @@ -525,11 +614,11 @@ object MakerChecker extends MdcLoggable { tryo { List( seed("DynamicResourceDoc", DynamicResourceDoc.findAll().filter(r => blank(r.ApprovedHash.get)))( - r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), + r => bodyHashOf(r.Lang.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), seed("DynamicMessageDoc", DynamicMessageDoc.findAll().filter(r => blank(r.ApprovedHash.get)))( - r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), + r => bodyHashOf(r.Lang.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), seed("ConnectorMethod", ConnectorMethod.findAll().filter(r => blank(r.ApprovedHash.get)))( - r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), + r => bodyHashOf(r.Lang.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), seed("AbacRule", AbacRule.findAll().filter(r => blank(r.ApprovedHash.get)))( r => sha256Hex(Option(r.RuleCode.get).getOrElse("")), (r, h) => { r.ApprovedHash(h).save; () }) ).mkString(", ") diff --git a/obp-api/src/test/scala/code/api/v4_0_0/ConnectorMethodTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/ConnectorMethodTest.scala index 666ff9b0e3..861c261865 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/ConnectorMethodTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/ConnectorMethodTest.scala @@ -104,7 +104,7 @@ class ConnectorMethodTest extends V400ServerSetup { .find(net.liftweb.mapper.By(code.connectormethod.ConnectorMethod.ConnectorMethodId, connectorMethod.connectorMethodId.getOrElse(""))) .openOrThrowException("stored connector method not found") storedConnectorMethod.CreatedByUserId.get should be (resourceUser1.userId) - storedConnectorMethod.MethodBodyHash.get should be (code.api.util.APIUtil.sha256Hex(postConnectorMethod.decodedMethodBody)) + storedConnectorMethod.MethodBodyHash.get should be (code.api.util.APIUtil.dynamicCodeHash(postConnectorMethod.programmingLang, postConnectorMethod.decodedMethodBody)) Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanCreateMethodRouting.toString) diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicMessageDocTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicMessageDocTest.scala index cc4265ebef..558d0dbe3a 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicMessageDocTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicMessageDocTest.scala @@ -102,7 +102,7 @@ class DynamicMessageDocTest extends V400ServerSetup { .find(net.liftweb.mapper.By(code.dynamicMessageDoc.DynamicMessageDoc.DynamicMessageDocId, dynamicMessageDoc.dynamicMessageDocId.getOrElse(""))) .openOrThrowException("stored dynamic message doc not found") storedMessageDoc.CreatedByUserId.get should be (resourceUser1.userId) - storedMessageDoc.MethodBodyHash.get should be (code.api.util.APIUtil.sha256Hex(postDynamicMessageDoc.decodedMethodBody)) + storedMessageDoc.MethodBodyHash.get should be (code.api.util.APIUtil.dynamicCodeHash(postDynamicMessageDoc.programmingLang, postDynamicMessageDoc.decodedMethodBody)) Then(s"we test the $ApiEndpoint2") diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala index bdb09c2a83..0210a3a5ce 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala @@ -399,7 +399,7 @@ class DynamicResourceDocTest extends V400ServerSetup { .find(net.liftweb.mapper.By(code.dynamicResourceDoc.DynamicResourceDoc.DynamicResourceDocId, docId)) .openOrThrowException("stored dynamic resource doc not found") storedRow.CreatedByUserId.get should be(resourceUser1.userId) - storedRow.MethodBodyHash.get should be(code.api.util.APIUtil.sha256Hex(posted.decodedMethodBody)) + storedRow.MethodBodyHash.get should be(code.api.util.APIUtil.dynamicCodeHash(posted.programmingLang, posted.decodedMethodBody)) When("We update the doc with a changed method body") val changedMethodBody = URLEncoder.encode( @@ -412,7 +412,7 @@ class DynamicResourceDocTest extends V400ServerSetup { Then("created_by_user_id is preserved, updated_by_user_id is recorded, and the hash reflects the new body") storedRow.CreatedByUserId.get should be(resourceUser1.userId) storedRow.UpdatedByUserId.get should be(resourceUser1.userId) - storedRow.MethodBodyHash.get should be(code.api.util.APIUtil.sha256Hex(URLDecoder.decode(changedMethodBody, "UTF-8"))) + storedRow.MethodBodyHash.get should be(code.api.util.APIUtil.dynamicCodeHash(posted.programmingLang, URLDecoder.decode(changedMethodBody, "UTF-8"))) } // Regression guard: rows created before the Lang column existed have a genuine SQL NULL there diff --git a/obp-api/src/test/scala/code/api/v7_0_0/CompileDynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/CompileDynamicResourceDocTest.scala new file mode 100644 index 0000000000..8c8e9dda03 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v7_0_0/CompileDynamicResourceDocTest.scala @@ -0,0 +1,160 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.v7_0_0 + +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.APIUtil.OAuth._ +import code.api.util.ApiRole +import code.api.util.ErrorMessages.DynamicCodeLangNotSupport +import code.api.v7_0_0.Http4s700.Implementations7_0_0 +import code.api.v7_0_0.JSONFactory700.DynamicResourceDocCompileJsonV700 +import code.entitlement.Entitlement +import code.setup.ServerSetupWithTestData +import com.github.dwickern.macros.NameOf.nameOf +import com.openbankproject.commons.util.ApiVersion +import org.json4s.JsonAST.{JArray, JBool, JInt} +import org.json4s.native.Serialization.write +import org.scalatest.Tag + +import java.net.URLEncoder + +/** + * This suite covers the v7.0.0 dry-run compile of a Dynamic Resource Doc method body + * (POST /management/dynamic-resource-docs/compile): the role gate, and the compiler diagnostics for + * each value of programming_lang, with line numbers relative to the body the author sent. + */ +class CompileDynamicResourceDocTest extends ServerSetupWithTestData { + + object VersionOfApi extends Tag(ApiVersion.v7_0_0.toString) + object ApiEndpoint1 extends Tag(nameOf(Implementations7_0_0.compileDynamicResourceDoc)) + + def compileRequest = (baseRequest / "obp" / "v7.0.0" / "management" / "dynamic-resource-docs" / "compile").POST + + // Braced body on purpose: .github/scripts/check_test_isolation.py only recognises `def name {` as a helper. + private def dynamicCodeOn(): Unit = { + setPropsValues("allow_user_generated_scala_code" -> "true") + } + + private def grantCreateRole(): Unit = + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + private def body(methodBody: String, programmingLang: Option[String]) = write(DynamicResourceDocCompileJsonV700( + request_verb = "POST", + request_url = "/compile_test/MY_USER_ID", + method_body = URLEncoder.encode(methodBody, "UTF-8"), + // The example bodies generate the case classes the Scala example body refers to; Java ignores them. + example_request_body = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.exampleRequestBody, + success_response_body = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.successResponseBody, + programming_lang = programmingLang + )) + + private val validScalaBody = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.decodedMethodBody + + // Line 9 of this body (the `return` line) refers to a variable that does not exist. + private val brokenJavaBody = + """package code.api.util.dynamic; + | + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class CompileTestBrokenJava implements Supplier> { + | @Override + | public Function get() { + | return args -> undefinedVariable; + | } + |} + |""".stripMargin + + private val validJavaBody = brokenJavaBody + .replace("CompileTestBrokenJava", "CompileTestValidJava") + .replace("undefinedVariable", "\"ok\"") + + feature("Compile Dynamic Resource Doc (dry run)") { + scenario("401 without a user, 403 without the create role", ApiEndpoint1, VersionOfApi) { + dynamicCodeOn() + makePostRequest(compileRequest, body(validScalaBody, None)).code should equal(401) + makePostRequest(compileRequest <@ (user1), body(validScalaBody, None)).code should equal(403) + } + + scenario("a Scala body compiles when programming_lang is omitted or Scala", ApiEndpoint1, VersionOfApi) { + dynamicCodeOn(); grantCreateRole() + for (lang <- List(None, Some("Scala"), Some("scala"))) { + val response = makePostRequest(compileRequest <@ (user1), body(validScalaBody, lang)) + withClue(s"programming_lang $lang, response ${response.body}: ") { + response.code should equal(200) + (response.body \ "compiles") should equal(JBool(true)) + } + } + + Given("a Scala body with a syntax error on its second line") + // A syntax error, not a type error: the Scala toolbox reports type errors without a position. + val broken = makePostRequest(compileRequest <@ (user1), body("val fine = 1\nval broken = )\nFuture.successful((fine, HttpCode.`200`(callContext.callContext)))", None)) + Then("the error is reported on line 2 of the body as sent") + withClue(s"response ${broken.body}: ") { + (broken.body \ "compiles") should equal(JBool(false)) + val errors = (broken.body \ "errors").asInstanceOf[JArray].arr + errors should not be empty + (errors.head \ "line") should equal(JInt(2)) + } + } + + scenario("a Java body is compiled by the Java compiler", ApiEndpoint1, VersionOfApi) { + dynamicCodeOn(); grantCreateRole() + Given("a valid Java body") + val valid = makePostRequest(compileRequest <@ (user1), body(validJavaBody, Some("Java"))) + Then("it compiles") + withClue(s"response ${valid.body}: ") { + valid.code should equal(200) + (valid.body \ "compiles") should equal(JBool(true)) + } + + Given("a Java body with an error on line 9") + val broken = makePostRequest(compileRequest <@ (user1), body(brokenJavaBody, Some("Java"))) + Then("the error is reported on line 9 of the body as sent, not on a line of the server's own") + withClue(s"response ${broken.body}: ") { + broken.code should equal(200) + (broken.body \ "compiles") should equal(JBool(false)) + val errors = (broken.body \ "errors").asInstanceOf[JArray].arr + errors should not be empty + (errors.head \ "line") should equal(JInt(9)) + (errors.head \ "message").values.toString should include("undefinedVariable") + } + + Given("the same valid Java body sent as Scala") + val asScala = makePostRequest(compileRequest <@ (user1), body(validJavaBody, Some("Scala"))) + Then("it does not compile: the language decides the compiler") + (asScala.body \ "compiles") should equal(JBool(false)) + } + + scenario("an unsupported programming_lang is rejected before anything is compiled", ApiEndpoint1, VersionOfApi) { + dynamicCodeOn(); grantCreateRole() + val response = makePostRequest(compileRequest <@ (user1), body(validScalaBody, Some("Cobol"))) + response.code should equal(400) + (response.body \ "message").values.toString should include(DynamicCodeLangNotSupport.takeWhile(_ != ':')) + } + } +} diff --git a/obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala index 832563ceec..09ef381885 100644 --- a/obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala +++ b/obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala @@ -43,7 +43,7 @@ import org.json4s.JsonAST.{JArray, JObject} import org.json4s.native.Serialization.write import org.scalatest.Tag -import java.net.URLDecoder +import java.net.{URLDecoder, URLEncoder} /** * Maker/checker for dynamic code (docs/MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md), phase 1, exercised @@ -186,7 +186,7 @@ class DynamicChangeRequestTest extends ServerSetupWithTestData { (ok.body \ "target_id").values.toString.nonEmpty should be(true) val row = storedDoc(doc.requestUrl).getOrElse(fail("doc not applied")) row.CreatedByUserId.get should be(resourceUser1.userId) - row.MethodBodyHash.get should be(APIUtil.sha256Hex(URLDecoder.decode(doc.methodBody, "UTF-8"))) + row.MethodBodyHash.get should be(APIUtil.dynamicCodeHash(doc.programmingLang, URLDecoder.decode(doc.methodBody, "UTF-8"))) row.ApprovedHash.get should be(row.MethodBodyHash.get) row.IsActive.get should be(true) MakerChecker.isExecutableDynamicResourceDoc(row.DynamicResourceDocId.get) should be(true) @@ -220,12 +220,27 @@ class DynamicChangeRequestTest extends ServerSetupWithTestData { callDynamicEndpoint("guard").codeIs(200) val row = storedDoc(doc.requestUrl).getOrElse(fail("doc not applied")) - When("the body hash is changed behind the API's back") + When("the stored hash column is changed behind the API's back") row.MethodBodyHash("tampered").save + Then("nothing changes: the guard recomputes the hash from the row, it does not trust the column") + MakerChecker.isExecutableDynamicResourceDoc(row.DynamicResourceDocId.get) should be(true) + val approvedBody = row.MethodBody.get + val approvedLang = row.Lang.get + + When("the body is changed behind the API's back") + row.MethodBody(approvedBody + URLEncoder.encode("\n// edited in the database\n", "UTF-8")).save Then("the endpoint is no longer served") MakerChecker.isExecutableDynamicResourceDoc(row.DynamicResourceDocId.get) should be(false) callDynamicEndpoint("guard").codeIs(404) - row.MethodBodyHash(row.ApprovedHash.get).save + row.MethodBody(approvedBody).save + callDynamicEndpoint("guard").codeIs(200) + + When("only the language is changed behind the API's back") + row.Lang("Java").save + Then("the approval does not carry over: it was for this body as Scala") + MakerChecker.isExecutableDynamicResourceDoc(row.DynamicResourceDocId.get) should be(false) + callDynamicEndpoint("guard").codeIs(404) + row.Lang(approvedLang).save callDynamicEndpoint("guard").codeIs(200) When("a maker without the approver role tries to deactivate") @@ -269,7 +284,7 @@ class DynamicChangeRequestTest extends ServerSetupWithTestData { When("the instance boots with approval required for the first time") MakerChecker.seedApprovedHashesIfEnabled() Then("the row's current body is treated as approved and the seed is logged") - approvedHashOf(legacy.requestUrl) should equal(APIUtil.sha256Hex(URLDecoder.decode(legacy.methodBody, "UTF-8"))) + approvedHashOf(legacy.requestUrl) should equal(APIUtil.dynamicCodeHash(legacy.programmingLang, URLDecoder.decode(legacy.methodBody, "UTF-8"))) callDynamicEndpoint("legacy").codeIs(200) logProvider.isExecuted(MakerChecker.seedMigrationName) should be(true) @@ -285,6 +300,94 @@ class DynamicChangeRequestTest extends ServerSetupWithTestData { callDynamicEndpoint("late").codeIs(404) } + scenario("a Java Dynamic Resource Doc is compiled as Java when its change request is approved", ApiEndpoint4, VersionOfApi) { + enableMakerChecker(); makerRoles(); checkerRoles() + val javaBody = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class MakerCheckerJavaDoc implements Supplier> { + | private Object apply(Object[] args) { + | @SuppressWarnings("unchecked") + | Map pathParams = (Map) args[1]; + | Map response = new LinkedHashMap<>(); + | response.put("user_id_from_path", pathParams.get("MY_USER_ID") + "_from_java"); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + val doc = newDoc("java").copy(methodBody = URLEncoder.encode(javaBody, "UTF-8"), programmingLang = "Java") + + When("the maker submits a Java doc and a checker approves it") + val created = makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(doc)) + created.codeIs(202) + makePostRequest((v7 / "management" / "dynamic-change-requests" / str(created.body, "dynamic_change_request_id") / "approval").POST <@ (user2), + s"""{"payload_hash":"${str(created.body, "payload_hash")}"}""").codeIs(200) + + Then("the stored row is Java, its approval covers the language, and the endpoint runs the Java code") + val row = storedDoc(doc.requestUrl).getOrElse(fail("doc not applied")) + row.Lang.get should equal("Java") + row.ApprovedHash.get should equal(APIUtil.dynamicCodeHash("Java", javaBody)) + val call = callDynamicEndpoint("java") + call.codeIs(200) + str(call.body, "user_id_from_path") should equal("user-xyz_from_java") + } + + scenario("the one-off rehash carries over only approvals still valid for the row's body, and keeps pending requests fresh", VersionOfApi) { + enableMakerChecker(); makerRoles(); checkerRoles() + def approve(created: code.setup.APIResponse) = + makePostRequest((v7 / "management" / "dynamic-change-requests" / str(created.body, "dynamic_change_request_id") / "approval").POST <@ (user2), + s"""{"payload_hash":"${str(created.body, "payload_hash")}"}""").codeIs(200) + val valid = newDoc("rehashok") + approve(makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(valid))) + val edited = newDoc("rehashedited") + approve(makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(edited))) + + Given("rows hashed in the old form (body only), as an instance had them before the upgrade") + def oldFormHash(row: DynamicResourceDoc) = APIUtil.sha256Hex(URLDecoder.decode(row.MethodBody.get, "UTF-8")) + val validRow = storedDoc(valid.requestUrl).get + validRow.MethodBodyHash(oldFormHash(validRow)).ApprovedHash(oldFormHash(validRow)).save + val editedRow = storedDoc(edited.requestUrl).get + editedRow.MethodBodyHash(oldFormHash(editedRow)).ApprovedHash(oldFormHash(editedRow)).save + And("one of them was edited in the database after its approval") + editedRow.MethodBody(editedRow.MethodBody.get + URLEncoder.encode("\n// edited in the database\n", "UTF-8")).save + And("an update to the valid row is pending, queued against its old-form hash") + val put = makePutRequest((v4 / "management" / "dynamic-resource-docs" / validRow.DynamicResourceDocId.get).PUT <@ (user1), + write(valid.copy(dynamicResourceDocId = Some(validRow.DynamicResourceDocId.get), summary = "pending change"))) + put.codeIs(202) + val pendingId = str(put.body, "dynamic_change_request_id") + code.dynamicchangerequest.DynamicChangeRequest.find(By(code.dynamicchangerequest.DynamicChangeRequest.DynamicChangeRequestId, pendingId)) + .foreach(_.CurrentPayloadHash(oldFormHash(storedDoc(valid.requestUrl).get)).save) + callDynamicEndpoint("rehashok").codeIs(404) + + When("the instance boots with the language-aware hash for the first time") + code.migration.MigrationScriptLog.findAll(By(code.migration.MigrationScriptLog.Name, MakerChecker.rehashMigrationName)).foreach(_.delete_!) + MakerChecker.rehashDynamicCodeWithLanguage() + + Then("the still-valid approval is moved to the new form and the endpoint is served again") + val validAfter = storedDoc(valid.requestUrl).get + val newForm = APIUtil.dynamicCodeHash(validAfter.Lang.get, URLDecoder.decode(validAfter.MethodBody.get, "UTF-8")) + validAfter.MethodBodyHash.get should equal(newForm) + validAfter.ApprovedHash.get should equal(newForm) + callDynamicEndpoint("rehashok").codeIs(200) + And("the row edited after approval stays unexecutable") + MakerChecker.isExecutableDynamicResourceDoc(storedDoc(edited.requestUrl).get.DynamicResourceDocId.get) should be(false) + And("the pending request is not stale, so it can still be approved") + approve(put) + storedDoc(valid.requestUrl).get.Summary.get should equal("pending change") + And("the rehash is logged so it never runs again") + code.migration.MigrationScriptLogProvider.migrationScriptLogProvider.vend.isExecuted(MakerChecker.rehashMigrationName) should be(true) + } + scenario("an update is queued with the live hash; rejection needs a comment and leaves the target untouched", ApiEndpoint5, VersionOfApi) { enableMakerChecker(); makerRoles(); checkerRoles() val doc = newDoc("upd") diff --git a/obp-api/src/test/scala/code/api/v7_0_0/Http4s700RoutesTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/Http4s700RoutesTest.scala index ad2f80375a..8093a019d5 100644 --- a/obp-api/src/test/scala/code/api/v7_0_0/Http4s700RoutesTest.scala +++ b/obp-api/src/test/scala/code/api/v7_0_0/Http4s700RoutesTest.scala @@ -2788,7 +2788,7 @@ class Http4s700RoutesTest extends ServerSetupWithTestData { Some(resourceUser1.userId) ).openOrThrowException("seed dynamic resource doc") val docId = seeded.dynamicResourceDocId.getOrElse(fail("seeded id")) - val expectedHash = code.api.util.APIUtil.sha256Hex(seeded.decodedMethodBody) + val expectedHash = code.api.util.APIUtil.dynamicCodeHash(seeded.programmingLang, seeded.decodedMethodBody) When("Unauthenticated GET of the list") val (unauthCode, _, _) = makeHttpRequest("/obp/v7.0.0/management/dynamic-resource-docs") @@ -2834,7 +2834,7 @@ class Http4s700RoutesTest extends ServerSetupWithTestData { Some(resourceUser1.userId) ).openOrThrowException("seed connector method") val id = seeded.connectorMethodId.getOrElse(fail("seeded id")) - val expectedHash = code.api.util.APIUtil.sha256Hex(seeded.decodedMethodBody) + val expectedHash = code.api.util.APIUtil.dynamicCodeHash(seeded.programmingLang, seeded.decodedMethodBody) addEntitlement("", resourceUser1.userId, code.api.util.ApiRole.canGetConnectorMethod.toString) val (code200, json, _) = makeHttpRequest( @@ -2853,7 +2853,7 @@ class Http4s700RoutesTest extends ServerSetupWithTestData { Some(resourceUser1.userId) ).openOrThrowException("seed dynamic message doc") val id = seeded.dynamicMessageDocId.getOrElse(fail("seeded id")) - val expectedHash = code.api.util.APIUtil.sha256Hex(seeded.decodedMethodBody) + val expectedHash = code.api.util.APIUtil.dynamicCodeHash(seeded.programmingLang, seeded.decodedMethodBody) addEntitlement("", resourceUser1.userId, code.api.util.ApiRole.canGetDynamicMessageDoc.toString) val (code200, json, _) = makeHttpRequest(