diff --git a/OPEN_CORRIDOR_INTERFACE_C_PUBLISH_PLAN.md b/OPEN_CORRIDOR_INTERFACE_C_PUBLISH_PLAN.md index 7242e077bc..b24839e154 100644 --- a/OPEN_CORRIDOR_INTERFACE_C_PUBLISH_PLAN.md +++ b/OPEN_CORRIDOR_INTERFACE_C_PUBLISH_PLAN.md @@ -356,10 +356,11 @@ parts: > (`OBP-OUTGOING-SETTLEMENT-ACCOUNT` → `OBP-INCOMING-SETTLEMENT-ACCOUNT`), > writes `settled_by_transaction_ids` AND `settled_by_transaction_request_id` > (TR B's id — kept even at net zero) attributes on each covered promise, -> flips them COMPLETED, and enqueues the messages into the `OpenCorridorOutbox` -> Mapper table in the same request DB transaction. `OpenCorridorOutboxRelay` -> (Boot-started when `open_corridor_enabled`, `open_corridor.outbox_relay_interval` -> default 10s) publishes with exponential backoff and records each §4.2 reply: +> flips them COMPLETED, and enqueues the messages into the `MessageOutbox` +> Mapper table in the same request DB transaction. `MessageOutboxRelay` +> (Boot-started on every instance, relaying Open Corridor rows only when +> `open_corridor_enabled`; interval `message_outbox.relay_interval_seconds`, +> default 10s, formerly `open_corridor.outbox_relay_interval`) publishes with exponential backoff and records each §4.2 reply: > settlement rows stay PENDING through SUBMITTED/SETTLING (redelivery-as-polling, > §4.4) until FINAL; refutable business errors (COMMITMENT-MISMATCH etc.) go > STICKY for operator reconciliation, never swallowed. Fails fast pre-mutation diff --git a/docs/MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md b/docs/MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md index e208b367bd..374ae97098 100644 --- a/docs/MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md +++ b/docs/MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md @@ -125,8 +125,11 @@ request. Rows are never deleted; the table is the audit log. - `ApprovedHash` on DynamicResourceDoc, DynamicMessageDoc, ConnectorMethod, AbacRule. - `IsActive` (default `true`) on every runtime-loaded dynamic model that lacks it. -The runtime loads a row only if `IsActive` is true and, for code, only if `MethodBodyHash == -ApprovedHash`. When maker/checker is disabled for a target type the hash check is skipped. +The runtime loads a row only if `IsActive` is true and, for code, only if the row's code hash +equals `ApprovedHash`. The code hash is recomputed from the row (its programming language and its +decoded method body, `APIUtil.dynamicCodeHash`) rather than read from the stored `MethodBodyHash`, so +an edit made directly in the database, to the body or to the language, withdraws the approval. When +maker/checker is disabled for a target type the hash check is skipped. ## 4. Endpoints (v7.0.0) @@ -193,8 +196,14 @@ fail loudly. On approval the server, in order: Rejection and withdrawal take `{ "comment": "…" }`. Withdrawal is by the requestor only. *(impl)* Hashes are bare SHA-256 hex, matching the existing `MethodBodyHash` column; a `sha256:` prefix is accepted on approval. `payload_hash` is over the canonical JSON of the request body; -`current_payload_hash` and `ApprovedHash` are the target's decoded method body hash (for ABAC -rules, the hash of `rule_code`). +`current_payload_hash` and `ApprovedHash` are the target's code hash: SHA-256 of its programming +language (trimmed, lower case, blank meaning `scala`), a newline, and its decoded method body (for +ABAC rules, the hash of `rule_code`). The language is included because the same text can be stored +as Scala, Java or Javascript and is compiled by the language's compiler, so approving a body as one +language must not approve it as another. The language version is not included: it belongs to the +deployed runtime, and including it would withdraw every approval at each compiler upgrade. Until +2026-10 the hash covered the body only; `MakerChecker.rehashDynamicCodeWithLanguage` moves existing +rows once at boot, carrying over only the approvals still valid for the row's current body. ### Reading diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index 303b96f8a5..5192975638 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -1857,7 +1857,6 @@ dynamic_code_allowed_obp_methods=[\ ErrorMessages.getClass.getTypeName -> "*",\ ExecutionContext.Implicits.getClass.getTypeName -> "global",\ JSONFactory400.getClass.getTypeName -> "createBanksJson",\ - classOf[Sandbox].getTypeName -> "runInSandbox",\ classOf[CallContext].getTypeName -> "*",\ classOf[ResourceDoc].getTypeName -> "getPathParams",\ "scala.reflect.runtime.package\$" -> "universe",\ diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index 869a7a1b9c..070bc35719 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -315,6 +315,10 @@ class Boot extends MdcLoggable { // Please note that migration scripts are executed after Lift Mapper Schemifier Migration.database.executeScripts(startedBeforeSchemifier = false) + // Code hashes cover the programming language as well as the body; move existing rows (and the + // approvals still valid for them) to that form once. Must run before the seed below. + code.dynamicchangerequest.MakerChecker.rehashDynamicCodeWithLanguage() + // Maker/checker for dynamic code: when first enabled, pre-existing code rows get their current // body hash recorded as approved so enabling the feature does not silently disable them. code.dynamicchangerequest.MakerChecker.seedApprovedHashesIfEnabled() diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index 30199ea5f9..e684eb40d8 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -408,7 +408,12 @@ object Constant extends MdcLoggable { final val CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL: Int = APIUtil.getPropsValue(s"createLocalisedResourceDocJson.cache.ttl.seconds", "3600").toInt final val GET_DYNAMIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"dynamicResourceDocsObp.cache.ttl.seconds", "3600").toInt final val GET_STATIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"staticResourceDocsObp.cache.ttl.seconds", "3600").toInt - final val SHOW_USED_CONNECTOR_METHODS: Boolean = APIUtil.getPropsAsBoolValue(s"show_used_connector_methods", false) + // def, not final val: DynamicUtil.Validation.validateDependency (dynamic-code dependency + // checking) needs this to react to a props change without a restart -- e.g. test-time + // setPropsValues overrides. A final val here would freeze at whatever value was true the + // moment this object was first touched (typically during server boot, well before any test + // scenario runs), and no later prop override could ever reach it. + def SHOW_USED_CONNECTOR_METHODS: Boolean = APIUtil.getPropsAsBoolValue(s"show_used_connector_methods", false) // Rate Limiting Cache Prefixes (with global namespace and versioning) // Both call_counter and rl_active are versioned for consistent cache invalidation diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala index e3e3e99191..0df1554632 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicEndpoints.scala @@ -33,7 +33,7 @@ import code.api.dynamic.endpoint.helper.practise.{DynamicEndpointCodeGenerator, import code.api.dynamic.endpoint.helper.practise.PractiseEndpointGroup import code.api.util.DynamicUtil.{DynamicCodeBody, Validation} import code.api.util.APIUtil.{BooleanBody, DoubleBody, EmptyBody, LongBody, Http4sEndpointIO, PrimaryDataBody, ResourceDoc, StringBody, getDisabledEndpointOperationIds} -import code.api.util.{CallContext, DynamicUtil} +import code.api.util.{APIUtil, CallContext, DynamicUtil} import net.liftweb.common.{Box, Failure, Full} import org.json4s.{JNothing, JValue} import org.json4s.JsonAST.{JBool, JDouble, JInt, JString} @@ -105,6 +105,17 @@ trait EndpointGroup { * @param methodBody it is url-encoded string for the api level code. */ object CompiledObjects { + /** + * This is the set of `programming_lang` values a Dynamic Resource Doc may have, compared after + * APIUtil.normaliseDynamicCodeLanguage (trimmed, lower case, blank meaning Scala). Create, update, + * validate and the dry-run compile all check against it, and CompiledObjects chooses its compiler + * from the same normalised value, so nothing that passes the check can reach the wrong compiler. + */ + val supportedLanguages: List[String] = List("scala", "java") + def isSupportedLanguage(programmingLang: String): Boolean = + supportedLanguages.contains(APIUtil.normaliseDynamicCodeLanguage(programmingLang)) + val supportedLanguagesText: String = "Scala, Java" + /** * The native http4s template a method body is inlined into. Returns the full source and the * 1-based line on which the method body starts, so compiler positions can be mapped back to it. @@ -153,10 +164,18 @@ object CompiledObjects { /** * Dry run without constructing a CompiledObjects (whose constructor compiles for real): compiler - * diagnostics with line numbers relative to the method body the author wrote. Empty = compiles. + * diagnostics with line numbers relative to the method body the author wrote, in the body's + * programming language (Scala or Java). Empty = compiles. */ - def compileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String): List[DynamicUtil.CompileProblem] = { + def compileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String, programmingLang: String = "Scala"): List[DynamicUtil.CompileProblem] = { val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8") + // Java bodies are compiled as written (no template, no generated case classes), so the example + // bodies play no part, as they play none in CompiledObjects' own Java branch. + if (APIUtil.normaliseDynamicCodeLanguage(programmingLang) == "java") DynamicUtil.checkJavaCode(decodedMethodBody) + else scalaCompileProblems(exampleRequestBody, successResponseBody, decodedMethodBody) + } + + private def scalaCompileProblems(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], decodedMethodBody: String): List[DynamicUtil.CompileProblem] = { val requestBody: Product = exampleRequestBody match { case Some(JString(s)) if StringUtils.isBlank(s) => toCaseObject(None) case _ => toCaseObject(exampleRequestBody) @@ -186,7 +205,7 @@ object CompiledObjects { } } -case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String) { +case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBody: Option[JValue], methodBody: String, programmingLang: String = "Scala") { val decodedMethodBody = URLDecoder.decode(methodBody, "UTF-8") val requestBody: Product = exampleRequestBody match { //this case means, we accept the empty string "" from json post body, we need to map it to None. @@ -196,7 +215,24 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo } val successResponse: Product = toCaseObject(successResponseBody) - private val partialFunction: Http4sEndpointIO = { + private val partialFunction: Http4sEndpointIO = APIUtil.normaliseDynamicCodeLanguage(programmingLang) match { + case "java" => + DynamicUtil.createJavaHttp4sEndpoint(decodedMethodBody) match { + case Full(func) => func + case Failure(msg: String, exception: Box[Throwable], _) => + throw exception.getOrElse(new RuntimeException(msg)) + case _ => throw new RuntimeException("compiled code return nothing") + } + case _ /* "scala", the default; create and update reject anything else (isSupportedLanguage) */ => + scalaPartialFunction + } + + // Unchanged Scala-template compile path, factored out so the `partialFunction` match above stays + // readable. Only evaluated for Scala-language docs (the default) — Java-language docs never + // touch this, so example/response-body JValues that don't fit the Scala case-class generator + // (irrelevant for Java, since it doesn't use RequestRootJsonClass/ResponseRootJsonClass) are a + // non-issue there. + private def scalaPartialFunction: Http4sEndpointIO = { //If the requestBody is PrimaryDataBody, return None. otherwise, return the exampleRequestBody:Option[JValue] // In side OBP resourceDoc, requestBody and successResponse must be Product type， @@ -237,15 +273,28 @@ case class CompiledObjects(exampleRequestBody: Option[JValue], successResponseBo * this will check all the dynamic scala code dependencies at compile time. * *Search for the usage, you can see how to use it in OBP code. + * + * Scala-only: for the Scala language, `this.partialFunction` IS the compiled user code, so + * validating its bytecode directly is correct. For Java, `this.partialFunction` is instead + * OBP's own Http4sEndpointIO wrapper (built by DynamicUtil.createJavaHttp4sEndpoint) around the + * real compiled Java class -- its bytecode legitimately calls internal OBP helpers + * (DynamicUtil.javaValueToJValue/logger, CustomJsonFormats.formats, JsonAliases.compactRender) + * that were never meant to be dependency-whitelisted, since they are framework glue, not + * user-supplied code. createJavaHttp4sEndpoint already validates the real compiled Java class + * internally (see its own doc comment) before ever returning that wrapper, so re-validating the + * wrapper here is both redundant and wrong -- it would reject every Java doc unconditionally. */ - def validateDependency() = Validation.validateDependency(this.partialFunction) + def validateDependency() = APIUtil.normaliseDynamicCodeLanguage(programmingLang) match { + case "java" => () + case _ => Validation.validateDependency(this.partialFunction) + } /** * Dry run: compiler diagnostics for this body, with line numbers relative to the method body the * author wrote (the wrapper's own lines are subtracted). Empty = compiles. Nothing is evaluated or cached. */ def compileProblems(): List[DynamicUtil.CompileProblem] = - CompiledObjects.compileProblems(exampleRequestBody, successResponseBody, methodBody) + CompiledObjects.compileProblems(exampleRequestBody, successResponseBody, methodBody, programmingLang) /** * Wraps the compiled partial function as an endpoint. This used to bind a per-bank diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala index 0a15a2f511..2c52f6eb8e 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/helper/DynamicResourceDocsEndpointGroup.scala @@ -52,6 +52,22 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel try { Some(toResourceDoc(dynamicDoc)) } catch { + // Validation.validateDependency / createJavaHttp4sEndpoint's own rejection path both throw + // this specifically for a dependency-whitelist miss -- distinct from a genuine compile + // failure, and reachable here (not just at create/update time) because CompiledObjects' + // validation runs fresh on every construction and dynamic_code_allowed_obp_methods + // can be tightened after a doc was already registered. Logging it as a "deprecated Lift + // contract" problem sends whoever reads this log to re-author a body that is not the + // problem, instead of at the whitelist they (or someone else) just edited. + case e: code.api.JsonResponseException => + val reason = e.jsonResponse match { + case APIUtil.JsonResponseExtractor(msg, _) => msg + case _ => Option(e.getMessage).getOrElse("") + } + logger.error(s"[DynamicResourceDocsEndpointGroup] skipping dynamic resource doc '${dynamicDoc.requestVerb} ${dynamicDoc.requestUrl}' " + + s"(id=${dynamicDoc.dynamicResourceDocId.getOrElse("")}, programming_lang=${dynamicDoc.programmingLang}): rejected by dependency " + + s"validation (dynamic_code_allowed_obp_methods). $reason") + None case e: Throwable => logger.error(s"[DynamicResourceDocsEndpointGroup] skipping dynamic resource doc '${dynamicDoc.requestVerb} ${dynamicDoc.requestUrl}' " + s"(id=${dynamicDoc.dynamicResourceDocId.getOrElse("")}): its methodBody could not be compiled under the native http4s contract. " + @@ -80,7 +96,7 @@ object DynamicResourceDocsEndpointGroup extends EndpointGroup with code.util.Hel * */ private val toResourceDoc: JsonDynamicResourceDoc => ResourceDoc = { dynamicDoc => - val compiledObjects = CompiledObjects(dynamicDoc.exampleRequestBody, dynamicDoc.successResponseBody, dynamicDoc.methodBody) + val compiledObjects = CompiledObjects(dynamicDoc.exampleRequestBody, dynamicDoc.successResponseBody, dynamicDoc.methodBody, dynamicDoc.programmingLang) ResourceDoc( // partialFunction is a no-op stub — the runtime dispatch uses the native handler in // dynamicHttp4sFunction (the compiled artifact is OBPEndpointIO, not the Lift OBPEndpoint). diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index ff0b13822e..a281069569 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -3410,6 +3410,29 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ digest.map(b => f"$b%02x").mkString } + /** + * This is the hash stored as `MethodBodyHash` on runtime-compiled code (Dynamic Resource Docs, + * Dynamic Message Docs, Connector Methods), and therefore the value a maker/checker approval binds + * to (`ApprovedHash`). + * + * It covers the programming language as well as the body. The same text can be stored as Scala, + * Java or Javascript, and the runtime picks a compiler from the language, so an approval of a body + * as one language must not carry over when only the language is changed. The language is + * normalised (trimmed, lower case, blank meaning `scala`, the default every one of these types + * applies), because the runtime treats `Java` and `java` as the same language. + * + * The language version is deliberately not included: it is a property of the deployed runtime, not + * of the row, and including it would withdraw every approval at each compiler upgrade. + * + * Until 2026-10 the hash covered the body only; `MakerChecker.rehashDynamicCodeWithLanguage` moves + * existing rows to this form once. + */ + def dynamicCodeHash(programmingLang: String, decodedMethodBody: String): String = + sha256Hex(normaliseDynamicCodeLanguage(programmingLang) + "\n" + Option(decodedMethodBody).getOrElse("")) + + def normaliseDynamicCodeLanguage(programmingLang: String): String = + Option(programmingLang).map(_.trim.toLowerCase).filter(_.nonEmpty).getOrElse("scala") + /** * Create the explicit CounterpartyId, (Used in `Create counterparty for an account` endpoint ). * This is just a UUID, use both in Counterparty.counterpartyId and CounterpartyMetadata.counterpartyId @@ -4526,8 +4549,10 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * * than the return value may be (getUserAndSessionContextFuture, ***,***),(map,***,***), (getOrElse,***,***) ...... */ - def getDependentMethods(className: String, methodName:String, signature: String): List[(String, String, String)] = { - if (SHOW_USED_CONNECTOR_METHODS) { + // force bypasses the SHOW_USED_CONNECTOR_METHODS gate below -- see + // DynamicUtil.getDynamicCodeDependentMethods' doc comment for why security validation needs this. + def getDependentMethods(className: String, methodName:String, signature: String, force: Boolean = false): List[(String, String, String)] = { + if (SHOW_USED_CONNECTOR_METHODS || force) { val methods = ListBuffer[(String, String, String)]() //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. //eg: className == code.api.UKOpenBanking.v3_1_0.APIMethods_AccountAccessApi$$anonfun$createAccountAccessConsents$lzycompute$1 diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index bf6b115e75..b7a7ec2694 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -84,23 +84,95 @@ object DynamicUtil extends MdcLoggable{ /** * Compile `code` for diagnostics only: nothing is evaluated, nothing is cached. Empty list = compiles. * Toolboxes are not thread-safe, so checks are serialised. Callers must apply the kill switch and a role. + * + * The code is checked inside a method, because that is how it is really compiled: ToolBox.compile + * wraps the tree in a method before compiling it. Checked bare, a `return` in the code (which the + * documented example method body uses to answer early with an error) is reported as "return + * outside method definition", although a real create accepts it. The wrapper is added on the same + * line as the start of the code, so line numbers are unchanged; columns on that first line are + * corrected for it. */ def checkScalaCode(code: String): List[CompileProblem] = checkToolBox.synchronized { checkFrontEnd.reset() + val wrapperStart = "{ def dryRunCompileWrapper(): Any = { " + val wrapperEnd = "\n}; () }" val failure: Option[String] = try { - checkToolBox.typecheck(checkToolBox.parse(code)) + checkToolBox.typecheck(checkToolBox.parse(wrapperStart + code + wrapperEnd)) None } catch { case e: ToolBoxError => Some(e.message) } val collected = checkFrontEnd.infos.toList.filter(_.severity == checkFrontEnd.ERROR).map { info => val (line, column) = if (info.pos != null && info.pos.isDefined) (info.pos.line, info.pos.column) else (0, 0) - CompileProblem(line, column, "ERROR", info.msg) + val columnInCode = if (line == 1 && column > wrapperStart.length) column - wrapperStart.length else column + CompileProblem(line, columnInCode, "ERROR", info.msg) } if (collected.nonEmpty) collected else failure.map(m => CompileProblem(0, 0, "ERROR", m.stripPrefix("reflective typecheck has failed:").stripPrefix("reflective compilation has failed:").trim)).toList } + + /** + * This compiles a Java `method_body` for diagnostics only, the Java counterpart of + * [[checkScalaCode]]. Empty list = compiles. Line numbers are relative to the body the author + * wrote. + * + * The source is prepared exactly as createJavaHttp4sEndpoint prepares it (the author's `package` + * line replaced by a fresh generated package), and compiled by the same system Java compiler with + * the same in-memory file manager, so it reports what a real create would report. Unlike the real + * compile, nothing is loaded or constructed: java-scriptengine's own compile also instantiates the + * class, which would run the author's constructor and static initialisers. Nothing is cached. + * Callers must apply the kill switch and a role. + */ + def checkJavaCode(methodBody: String): List[CompileProblem] = { + import javax.tools.{Diagnostic, DiagnosticCollector, JavaFileObject, ToolProvider} + import scala.jdk.CollectionConverters._ + + val compiler = ToolProvider.getSystemJavaCompiler + if (compiler == null) { + List(CompileProblem(0, 0, "ERROR", "No Java compiler is available: this server runs on a Java runtime without the compiler (a JRE rather than a JDK).")) + } else { + // One line is added above the author's code: the generated package statement. + val linesAdded = 1 + val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) + val packageName = "code.api.util.dynamic." + UUID.randomUUID().toString.replaceAll("^|-", "_") + val javaCode = s"package $packageName;\n${packageMatcher.replaceFirst("")}\n" + + (Box tryo { new ch.obermuhlner.scriptengine.java.name.DefaultNameStrategy().getFullName(javaCode) }) match { + case Full(fullClassName) => + val diagnostics = new DiagnosticCollector[JavaFileObject] + val fileManager = new ch.obermuhlner.scriptengine.java.MemoryFileManager( + compiler.getStandardFileManager(diagnostics, null, null), null) + val simpleClassName = StringUtils.substringAfterLast("." + fullClassName, ".") + val source = fileManager.createSourceFileObject(null, simpleClassName, javaCode) + val compiled: Boolean = compiler.getTask(null, fileManager, diagnostics, null, null, java.util.Arrays.asList(source)).call() + val problems = diagnostics.getDiagnostics.asScala.toList.filter(_.getKind == Diagnostic.Kind.ERROR).map { d => + val line = if (d.getLineNumber > linesAdded) (d.getLineNumber - linesAdded).toInt else 0 + val column = if (line > 0 && d.getColumnNumber > 0) d.getColumnNumber.toInt else 0 + CompileProblem(line, column, "ERROR", d.getMessage(java.util.Locale.ENGLISH)) + } + if (problems.nonEmpty || compiled) problems + else List(CompileProblem(0, 0, "ERROR", "The Java compiler rejected the method body without reporting why.")) + case failure => + // DefaultNameStrategy fails when there is no class declaration to name the source file after. + val reason = failure match { case f: Failure => f.msg; case _ => "no class declaration found" } + List(CompileProblem(0, 0, "ERROR", s"A Java method body must declare a public class implementing Supplier>: $reason")) + } + } + } + // Neither this nor memoJavaCompiledScript below ever evicts, so each distinct ClassLoader (and + // therefore each distinct compiled Java method_body -- java-scriptengine hands createJavaHttp4sEndpoint + // a fresh MemoryClassLoader per compile) is retained for the life of the process, along with its + // ClassPool. This is the same unbounded-but-trusted-operator-only tradeoff dynamicCompileResult + // below already makes for the Scala compile cache, predating the Java path: registering a dynamic + // resource doc is gated behind canCreateDynamicResourceDoc / canCreateBankLevelDynamicResourceDoc, + // not open to arbitrary callers, and a served endpoint's ClassLoader must stay reachable for as + // long as that endpoint keeps serving requests -- an eviction policy here would need to be + // reference-counted against currently-registered docs to avoid reclaiming a live one, which is a + // larger change than this cache's existing (pre-Java) design accounted for. private val memoClassPool = new Memo[ClassLoader, ClassPool] + // Caches only the compiled artifact (deterministic given the source string), never the + // validation outcome built on top of it -- see createJavaHttp4sEndpoint's doc comment. + private val memoJavaCompiledScript = new Memo[String, Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript]] private def getClassPool(classLoader: ClassLoader) = memoClassPool.memoize(classLoader){ val cp = ClassPool.getDefault @@ -224,29 +296,71 @@ object DynamicUtil extends MdcLoggable{ } /** - * NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. + * NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. * @param clazz * @param predicate + * @param force bypasses the SHOW_USED_CONNECTOR_METHODS gate below. SHOW_USED_CONNECTOR_METHODS + * exists to opt in to an unrelated, expensive introspection/reporting feature (which + * connector methods a static endpoint touches) — it was never meant to gate SECURITY + * validation, which reuses this same bytecode scan. Without `force`, a deployment + * that sets dynamic_code_obp_calls_are_restricted=true (the documented, security- + * relevant prop) but leaves the unrelated show_used_connector_methods at its default + * false would silently get an always-empty dependency list here — every dynamic-code + * call looks "allowed" no matter what it does, because there is nothing to check + * against the whitelist. Validation.validateDependency passes force=true so it is + * controlled solely by dynamic_code_obp_calls_are_restricted, matching what an + * operator following that prop's own documentation would expect. * @return */ - def getDynamicCodeDependentMethods(clazz: Class[_], predicate: String => Boolean = _ => true): List[(String, String, String)] = - if (SHOW_USED_CONNECTOR_METHODS) { + def getDynamicCodeDependentMethods(clazz: Class[_], predicate: String => Boolean = _ => true, force: Boolean = false): List[(String, String, String)] = + if (SHOW_USED_CONNECTOR_METHODS || force) { val className = clazz.getTypeName val listBuffer = new ListBuffer[(String, String, String)]() val classPool = getClassPool(clazz.getClassLoader) - //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. + //NOTE: MEMORY_USER this ctClass will be cached in ClassPool, it may load too many classes into heap. val ctClass = classPool.get(className) + + // A same-class (or same-generated-unit, for the Scala nested-closure case below) call is not + // itself a dependency to police -- recurse into what the TARGET method calls instead of + // flagging the call itself as forbidden, all the way down until a genuinely foreign + // dependency is reached. This is required for Java: every Java dynamic resource doc + // implements Supplier> (the documented convention), and the + // compiler always erases that generic Supplier.get() to a synthetic bridge method + // `Object get()` whose body is just `return this.get();` -- an ordinary same-class + // invokevirtual call to the real, properly-typed get(). A single level of unrolling only + // fixes that one hop: any Java body that factors logic into its own private helper methods + // (an entirely normal thing to do) reintroduces the exact same false rejection one level + // deeper, since the un-recursed helper's own callees would otherwise be appended as raw + // (thisClass, method) tuples and then rejected as calls to an unwhitelistable random-UUID + // class. `visited` guards against a call cycle -- direct or mutual recursion between + // same-class private methods (e.g. a fibonacci/factorial helper) is entirely normal Java and + // would otherwise recurse forever. On hitting a cycle this contributes nothing further (Nil), + // not a leaf: the recursive call is still a same-class call, not a foreign dependency, and + // whatever it in turn depends on is already being expanded by the in-progress call further up + // this same path -- returning it as a leaf here would flag the method's own name + // (unwhitelistable, like any other randomly-named dynamic class) as a forbidden dependency, + // exactly the bug this whole function exists to avoid. + def expand(typeName: String, methodName: String, signature: String, visited: Set[(String, String, String)]): List[(String, String, String)] = { + val key = (typeName, methodName, signature) + val sameUnit = typeName == className || + (className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName + "$")) + if (!sameUnit) { + List(key) + } else if (visited.contains(key)) { + Nil + } else { + APIUtil.getDependentMethods(typeName, methodName, signature, force).flatMap { case (t, m, s) => + expand(t, m, s, visited + key) + } + } + } + for { method <- ctClass.getDeclaredMethods.toList if predicate(method.getName) - ternary @ (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature) + (typeName, methodName, signature) <- APIUtil.getDependentMethods(className, method.getName, method.getSignature, force) } yield { - // if method is also dynamic compile code, extract it's dependent method - if(className.startsWith(typeName) && methodName.startsWith(clazz.getPackage.getName+ "$")) { - listBuffer.appendAll(APIUtil.getDependentMethods(typeName, methodName, signature)) - } else { - listBuffer.append(ternary) - } + listBuffer.appendAll(expand(typeName, methodName, signature, Set.empty)) } listBuffer.distinct.toList @@ -353,6 +467,14 @@ object DynamicUtil extends MdcLoggable{ object Validation { + // def, not val, throughout this object: these must react to a props change (e.g. test-time + // setPropsValues) without a restart, not freeze at whatever the props held the moment + // Validation was first touched (typically by whichever dynamic-code test happens to run + // first in a shared test JVM). This costs nothing extra in production -- the only expensive + // step, DynamicUtil.compileScalaCodeUnchecked, is already memoized by the exact source + // string, so re-evaluating these on every call is a cache hit unless the underlying props + // value actually changed. + /** * Turn the `dynamic_code_allowed_obp_methods` props value into the Scala source * that, once compiled, yields the whitelist. @@ -373,7 +495,7 @@ object DynamicUtil extends MdcLoggable{ dependenciesString.replaceFirst("\\[", "Map[String, String](").dropRight(1) + ").mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet).toMap" - + // Runtime permission control was removed with the sandbox: SecurityManager is gone // from JDK 24+ (JEP 486) and OBP requires JVM 25, so it could not be enforced. See // DynamicUtil.DynamicCodeBody for what replaced it and which controls still work. @@ -385,22 +507,24 @@ object DynamicUtil extends MdcLoggable{ * control is an allowlist of the OBP methods dynamic code may call. * * Both legacy names are still read, because dropping them would silently disable a restriction - * an operator had deliberately turned on. A deployment using either is warned, once, at boot. + * an operator had deliberately turned on. A deployment using either is warned once per name; + * this is read on every validation (see "def, not val" above), so the warning must not repeat. */ + private val legacyPropsWarned = java.util.concurrent.ConcurrentHashMap.newKeySet[String]() private def legacyProp(current: String, legacy: String): Box[String] = { val legacyValue = APIUtil.getPropsValue(legacy) - if (legacyValue.isDefined && APIUtil.getPropsValue(current).isEmpty) { + if (legacyValue.isDefined && APIUtil.getPropsValue(current).isEmpty && legacyPropsWarned.add(legacy)) { logger.warn(s"Props `$legacy` is deprecated and has been renamed to `$current`. The old " + s"name is still honoured, but rename it: support will be removed.") } APIUtil.getPropsValue(current) or legacyValue } - val dependenciesString = + def dependenciesString = legacyProp("dynamic_code_allowed_obp_methods", "dynamic_code_compile_validate_dependencies") .openOr("[]").trim - val scalaCodeDependencies = dependenciesScalaCode(dependenciesString) - val dependenciesBox: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodeDependencies) + def scalaCodeDependencies = dependenciesScalaCode(dependenciesString) + def dependenciesBox: Box[Map[String, Set[String]]] = DynamicUtil.compileScalaCodeUnchecked(scalaCodeDependencies) /** * Compilation OBP Dependencies Guard, only checked the OBP methods, not scala/Java libraies(are checked during the runtime.). @@ -430,7 +554,7 @@ object DynamicUtil extends MdcLoggable{ // PractiseEndpoint.getClass.getTypeName + "*" -> "*", // // ).mapValues(v => StringUtils.split(v, ',').map(_.trim).toSet) - val allowedCompilationMethods: Map[String, Set[String]] = dependenciesBox.openOrThrowException("Can not compile the props `dynamic_code_allowed_obp_methods` to Map") + def allowedCompilationMethods: Map[String, Set[String]] = dependenciesBox.openOrThrowException("Can not compile the props `dynamic_code_allowed_obp_methods` to Map") //Do not touch this Set, try to use the `allowedPermissions` and `allowedMethods` to control the sandbox val restrictedTypes = Set( @@ -447,6 +571,11 @@ object DynamicUtil extends MdcLoggable{ * Here only validate the restricted types(isObpClass + val restrictedTypes), not all scala/java types. */ private def validateDependency(dependentMethods: List[(String, String, String)]) = { + // Bound once per call, not re-derived per dependency tuple: allowedCompilationMethods is a + // def (see the "def, not val" comment above) so it observes a live props change, but it + // recompiles the whitelist source on every access -- reading it twice per element inside + // the `collect` guard below would mean up to 2N re-derivations for N dependency tuples. + val allowedCompilationMethods = this.allowedCompilationMethods val notAllowedDependentMethods = dependentMethods collect { case (typeName, method, _) if isRestrictedType(typeName) && @@ -471,7 +600,9 @@ object DynamicUtil extends MdcLoggable{ val restricted = legacyProp("dynamic_code_obp_calls_are_restricted", "dynamic_code_compile_validate_enable") .map(_.trim.equalsIgnoreCase("true")).openOr(false) if(restricted){ - val dependentMethods: List[(String, String, String)] = DynamicUtil.getDynamicCodeDependentMethods(obj.getClass) + // force=true: this check must not also require the unrelated show_used_connector_methods + // prop -- see getDynamicCodeDependentMethods' doc comment for why. + val dependentMethods: List[(String, String, String)] = DynamicUtil.getDynamicCodeDependentMethods(obj.getClass, force = true) validateDependency(dependentMethods) } else{ // If false, nothing to do here. ; @@ -554,4 +685,196 @@ object DynamicUtil extends MdcLoggable{ } } } + + /** + * Converts a plain value returned by a compiled Java `method_body` into a JValue, for endpoints + * where json4s' `Extraction.decompose` cannot help: it works by Scala-case-class/collection + * reflection, so a `java.util.Map`/`java.util.List` returned from Java decomposes to `{}`/`[]` + * (its entries are invisible to Scala reflection) rather than throwing — a silent data-loss bug, + * not a compile or runtime error, so it only surfaces as an empty response body. Recurses through + * the Java collection types directly; anything else (including a Scala case class constructed + * from Java, as ConnectorMethod's Java example does) falls back to Extraction.decompose. + */ + private def javaValueToJValue(value: Any): JValue = { + import scala.jdk.CollectionConverters._ + value match { + case null => JNull + case jv: JValue => jv + case m: java.util.Map[_, _] => + JObject(m.asScala.toList.map { case (k, v) => (String.valueOf(k), javaValueToJValue(v)) }) + case l: java.util.List[_] => + JArray(l.asScala.toList.map(javaValueToJValue)) + case s: String => JString(s) + case b: java.lang.Boolean => JBool(b) + case i: java.lang.Integer => JInt(BigInt(i.intValue())) + case l: java.lang.Long => JInt(BigInt(l.longValue())) + case d: java.lang.Double => JDouble(d.doubleValue()) + case f: java.lang.Float => JDouble(f.doubleValue()) + case bd: java.math.BigDecimal => JDecimal(BigDecimal(bd)) + case other => Extraction.decompose(other)(CustomJsonFormats.formats) + } + } + + /** + * Compiles a Java `method_body` for a DynamicResourceDoc endpoint into a native + * `Http4sEndpointIO` (`PartialFunction[Request[IO], CallContext => IO[Response[IO]]]`), the same + * type the Scala template compiles to in DynamicEndpoints.CompiledObjects. + * + * Reuses the same JSR-223 "java" engine (backed by a real javax.tools.JavaCompiler via + * ch.obermuhlner:java-scriptengine — see createJavaFunction above) and the same + * package-uniquification trick, but — unlike createJavaFunction, whose DynamicFunction shape is + * specific to the ConnectorMethod feature — wraps the compiled function in a hand-written + * Http4sEndpointIO here in Scala. The Java method_body never has to construct cats.effect.IO, + * org.http4s.Response, or a Scala PartialFunction: it only ever returns a plain Java object + * (Map/List/String/number/boolean/etc.), which this adapter serializes via javaValueToJValue + * above (NOT Extraction.decompose directly — see that method's doc comment for why). + * + * Java-side convention (identical to the existing ConnectorMethod convention): the pasted class + * implements java.util.function.Supplier>. The + * compiled function is invoked with: + * args(0) = the raw request body (String, or null if the request had none) + * args(1) = path params (java.util.Map) + * args(2) = the CallContext (present whenever this endpoint is actually being served) + * mirroring createJavaFunction's own `func(args ++ cc)` call (line above): appending an + * Option[CallContext] via `++` appends its *contents* (0 or 1 raw CallContext), not the Option + * wrapper itself, so Java reads args[2] directly as a CallContext, no unwrapping needed. + * + * Unlike createJavaFunction, this validates the actual compiled Java class (not just its Scala + * wrapper) against `dynamic_code_allowed_obp_methods`/`dynamic_code_obp_calls_are_restricted`. + * CompiledObjects.validateDependency() (called by the ResourceDoc-creation flow) only ever sees + * `this.partialFunction` — the hand-written Http4sEndpointIO below — whose own bytecode just + * calls `java.util.function.Function.apply`, a non-restricted type; it can't see what the pasted + * Java class does inside apply(Object[]). Worse, `func` itself (the Function returned by the + * pasted class's get()) is commonly a method reference (`this::apply`), which the JVM + * materialises as a synthetic lambda class whose bytecode is just a delegating call — validating + * `func.getClass` would be equally blind. So we go through the JSR-223 Compilable API directly + * (JavaScriptEngine implements it) instead of plain eval(), to get the real top-level compiled + * class/instance (JavaCompiledScript.getCompiledClass/getCompiledInstance) and validate that + * before the function is ever returned or invoked. + */ + def createJavaHttp4sEndpoint(methodBody: String): Box[code.api.util.APIUtil.Http4sEndpointIO] = + if (!dynamicCodeExecutionEnabled) Failure(ErrorMessages.DynamicCodeExecutionDisabled) + else { + import cats.effect.IO + import code.api.util.APIUtil.Http4sEndpointIO + import com.openbankproject.commons.ExecutionContext.Implicits.global + import com.openbankproject.commons.util.JsonAliases.compactRender + import org.http4s.headers.`Content-Type` + import org.http4s.dsl.io._ + import org.http4s.{MediaType, Request, Response} + + import scala.jdk.CollectionConverters._ + + // Only the compile step is memoized — deterministic given the same source string, and the + // one genuinely expensive part (a real javax.tools.JavaCompiler invocation). Dependency + // validation below is NOT memoized: it depends on mutable external config + // (dynamic_code_obp_calls_are_restricted/dynamic_code_allowed_obp_methods), which can change between two + // createJavaHttp4sEndpoint calls for the identical source string — e.g. a doc compiled once + // while validation was off, then a later create/update call resubmitting the exact same + // method_body after validation was turned on and the whitelist tightened. An earlier version + // of this function memoized the validated *result* (Box[Http4sEndpointIO]) as a single unit, + // so that second call silently reused the first call's unvalidated success — bypassing the + // now-stricter policy for any resubmitted source. Re-running validation on every call costs + // little: it is Javassist bytecode inspection plus a Map lookup, not another compile. + val compiledScriptBox: Box[ch.obermuhlner.scriptengine.java.JavaCompiledScript] = + memoJavaCompiledScript.memoize("java-http4s-endpoint:" + methodBody) { + // Real compile happens here (javax.tools.JavaCompiler via the JSR-223 "java" engine) — + // any Java syntax/type error surfaces as an exception, caught by this `Box tryo` and + // turned into a Failure. + Box tryo { + val packageExp = UUID.randomUUID().toString.replaceAll("^|-", "_") + val packageMatcher = Pattern.compile("""(?m)^\s*package\s+\S+?\s*;""").matcher(methodBody) + + val javaCode = s"""package code.api.util.dynamic.${packageExp}; + |${packageMatcher.replaceFirst("")} + |""".stripMargin + + val compiledScript = javaEngine.asInstanceOf[javax.script.Compilable].compile(javaCode) + .asInstanceOf[ch.obermuhlner.scriptengine.java.JavaCompiledScript] + + // getDynamicCodeDependentMethods loads a class's bytecode via Javassist's + // LoaderClassPath, which reads it through classLoader.getResourceAsStream(...). The + // compiler's ch.obermuhlner.scriptengine.java.MemoryClassLoader only overrides + // loadClass() — it never exposes the compiled bytes as a classpath resource — so that + // lookup silently fails (javassist.NotFoundException) and validation would see zero + // dependent methods no matter what the Java code actually calls. Read the bytes + // directly from the classloader's private byte map (reflection is unavoidable here: + // java-scriptengine exposes no public accessor) and hand them to Javassist explicitly + // via ByteArrayClassPath, so the real method bodies — including any restricted OBP + // call — are visible to validation. Done here, inside the compile memoization, so it + // runs exactly once per distinct source: ClassPool.appendClassPath has no dedup of its + // own, so doing this on every createJavaHttp4sEndpoint call (as an earlier version of + // this function did, on every resourceDocs-list rebuild for the process's lifetime) + // grew that ClassPool's classpath chain without bound. + val compiledClass = compiledScript.getCompiledClass + val classBytesField = compiledClass.getClassLoader.getClass.getDeclaredField("mapClassBytes") + classBytesField.setAccessible(true) + val classBytes = classBytesField.get(compiledClass.getClassLoader) + .asInstanceOf[java.util.Map[String, Array[Byte]]].get(compiledClass.getName) + // Fail loudly and specifically here rather than handing Javassist a null byte array -- + // that would only surface later, inside ByteArrayClassPath/ClassPool, as an opaque NPE + // with no indication that the cause was this reflective read (e.g. a java-scriptengine + // upgrade that changes mapClassBytes' keying from binary name to internal name, or that + // stops using that field name at all). + if (classBytes == null) { + throw new IllegalStateException( + s"createJavaHttp4sEndpoint: MemoryClassLoader.mapClassBytes has no entry for " + + s"${compiledClass.getName} -- java-scriptengine's internal layout may have changed") + } + getClassPool(compiledClass.getClassLoader) + .appendClassPath(new javassist.ByteArrayClassPath(compiledClass.getName, classBytes)) + + compiledScript + } + } + + // Deliberately outside compiledScriptBox's `Box tryo` AND outside the memoization above: a + // rejection here throws JsonResponseException, which must propagate UNCAUGHT (mirroring the + // Scala path's CompiledObjects.validateDependency(), also never wrapped in tryo) so + // compileDynamicResourceDoc's `case e: JsonResponseException => throw e` sees it intact. + // JsonResponseException never sets a Throwable message (getMessage == null); Box.tryo would + // catch it into Failure(null, Full(theException), Empty), and DynamicEndpoints.scala's + // `case Failure(msg: String, ...)` pattern silently fails to match a null msg — falling + // through to "compiled code return nothing" and discarding the real rejection reason. `.map` + // does not swallow exceptions the way `Box tryo` does, so this stays uncaught here. + compiledScriptBox.map { compiledScript => + // Validate the real compiled Supplier class before it's ever invoked — see the doc comment + // above for why this must run against getCompiledInstance, not `func`/`this.partialFunction`, + // and why it must run fresh on every call rather than being cached with the compile result. + Validation.validateDependency(compiledScript.getCompiledInstance) + + val func = compiledScript.eval().asInstanceOf[java.util.function.Function[Array[AnyRef], Any]] + val jsonContentType = `Content-Type`(MediaType.application.json) + + new Http4sEndpointIO { + override def isDefinedAt(req: Request[IO]): Boolean = true + + override def apply(req: Request[IO]): CallContext => IO[Response[IO]] = { cc => + val pathParams: java.util.Map[String, String] = cc.resourceDocument + .map(_.getPathParams(req.uri.path.segments.toList.map(_.encoded))) + .getOrElse(Map.empty[String, String]) + .asJava + + val valueIO: IO[Any] = IO.fromFuture(IO { + Future { + val args: Array[AnyRef] = Array(cc.httpBody.orNull, pathParams) + func(args ++ Some(cc)) + } + }) + + valueIO.flatMap { value => + Ok(compactRender(javaValueToJValue(value)), jsonContentType) + }.handleErrorWith { e => + logger.warn(s"createJavaHttp4sEndpoint: Java method_body threw", e) + InternalServerError( + compactRender(Extraction.decompose( + Map("code" -> 500, "message" -> s"OBP-50000: Unknown Error. ${e.getMessage}") + )(CustomJsonFormats.formats)), + jsonContentType + ) + } + } + } + } + } } diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 09c03a6db4..9a2bd7f4ab 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -4242,7 +4242,7 @@ object Glossary extends MdcLoggable { | |2) The checker reads the request (`GET /obp/v7.0.0/management/dynamic-change-requests/CHANGE_REQUEST_ID`, which returns the proposed and the current payload side by side) and approves it by quoting its `payload_hash`, the SHA-256 of the exact body, on `POST .../approval`. Only then is the change applied. OBP refuses an approval from the User who made the request (`OBP-30279`). | -|3) Content is approved, not records. Any later edit produces a new hash and needs a new approval. The runtime compiles and serves only rows whose body hash equals the hash a checker approved, so a row edited directly in the database does not run. +|3) Content is approved, not records. Any later edit produces a new hash and needs a new approval. The runtime compiles and serves only rows whose code hash equals the hash a checker approved. The code hash covers the programming language as well as the method body, and is recomputed from the row each time, so a row whose body or language is edited directly in the database does not run. | |4) Deactivating an artefact is a direct action by a single checker (`POST .../deactivation`), with no request: four eyes to enable, one pair to disable. Enabling it again goes through a request. | diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index 0f8bbb0edb..961ef238cf 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -9579,6 +9579,13 @@ object Http4s400 { } // Column widths: say which field is too long instead of letting the database answer OBP-50000. _ <- checkDynamicResourceDocFieldLengths(body, cc) + // Fail fast with a clean 400 before attempting compilation, rather than surfacing an + // unsupported programming_lang only as a generic DynamicCodeCompileFail. + _ <- code.util.Helper.booleanToFuture( + s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: ${code.api.dynamic.endpoint.helper.CompiledObjects.supportedLanguagesText}""", + cc = Some(cc)) { + code.api.dynamic.endpoint.helper.CompiledObjects.isSupportedLanguage(body.programmingLang) + } } yield () } @@ -9599,7 +9606,7 @@ object Http4s400 { private def compileDynamicResourceDoc(body: JsonDynamicResourceDoc, cc: CallContext): Unit = { try { - CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody).validateDependency() + CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency() } catch { case e: JsonResponseException => throw e case e: Exception => diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index eecb91d7ba..9c12faed85 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -4663,12 +4663,23 @@ object Http4s600 { case _ => true } } + // Mirrors Http4s400's validateDynamicResourceDocBody: fail fast on an unsupported + // programming_lang here too, rather than reporting `valid = true` for a language + // create would actually reject with 400 DynamicCodeLangNotSupport (CompiledObjects + // silently falls through to the Scala compile path for any value it doesn't + // recognise as Java, so an unsupported/misspelled language would otherwise still + // "validate" successfully as Scala). + _ <- Helper.booleanToFuture( + s"""$DynamicCodeLangNotSupport programming_lang ${body.programmingLang}, currently supported languages: ${code.api.dynamic.endpoint.helper.CompiledObjects.supportedLanguagesText}""", + cc = Some(cc)) { + code.api.dynamic.endpoint.helper.CompiledObjects.isSupportedLanguage(body.programmingLang) + } } yield try { code.api.dynamic.endpoint.helper.CompiledObjects( - body.exampleRequestBody, body.successResponseBody, body.methodBody).validateDependency() + body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang).validateDependency() ValidateDynamicResourceDocSuccessJsonV600( valid = true, - message = "Dynamic Resource Doc method body is valid Scala and uses allowed dependencies.") + message = s"Dynamic Resource Doc method body is valid ${body.programmingLang} and uses allowed dependencies.") } catch { case e: code.api.JsonResponseException => val errorText = e.jsonResponse match { diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index a96b026b97..1d07bac1fc 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -5893,7 +5893,7 @@ object Http4s700 { "GET", "/management/dynamic-resource-docs", "Get Dynamic Resource Docs (with provenance)", - s"""Returns all Dynamic Resource Docs, each wrapped with a `provenance` object recording who created / last updated the runtime-compiled code and a SHA-256 of its method body. + s"""Returns all Dynamic Resource Docs, each wrapped with a `provenance` object recording who created / last updated the runtime-compiled code and a SHA-256 of its programming language and method body. | |This is the v7.0.0 read view of the v4.0.0 Dynamic Resource Docs; create / update / delete remain on v4.0.0. | @@ -5951,7 +5951,7 @@ object Http4s700 { "GET", "/management/connector-methods", "Get Connector Methods (with provenance)", - s"""Returns all Connector Methods, each wrapped with a `provenance` object recording who created / last updated the runtime-compiled code and a SHA-256 of its method body. + s"""Returns all Connector Methods, each wrapped with a `provenance` object recording who created / last updated the runtime-compiled code and a SHA-256 of its programming language and method body. | |This is the v7.0.0 read view of the v4.0.0 Connector Methods; create / update remain on v4.0.0. | @@ -6009,7 +6009,7 @@ object Http4s700 { "GET", "/management/dynamic-message-docs", "Get Dynamic Message Docs (with provenance)", - s"""Returns all Dynamic Message Docs, each wrapped with a `provenance` object recording who created / last updated the runtime-compiled code and a SHA-256 of its method body. + s"""Returns all Dynamic Message Docs, each wrapped with a `provenance` object recording who created / last updated the runtime-compiled code and a SHA-256 of its programming language and method body. | |This is the v7.0.0 read view of the v4.0.0 Dynamic Message Docs; create / update / delete remain on v4.0.0. | @@ -6821,15 +6821,21 @@ object Http4s700 { _ <- code.util.Helper.booleanToFuture(s"""$InvalidJsonFormat The request_verb must be one of ["POST", "PUT", "GET", "DELETE"]""", cc = Some(cc)) { Set("POST", "PUT", "GET", "DELETE").contains(body.request_verb) } + programmingLang = body.programming_lang.getOrElse("Scala") + _ <- code.util.Helper.booleanToFuture( + s"""${code.api.util.ErrorMessages.DynamicCodeLangNotSupport} programming_lang $programmingLang, currently supported languages: ${code.api.dynamic.endpoint.helper.CompiledObjects.supportedLanguagesText}""", + cc = Some(cc)) { + code.api.dynamic.endpoint.helper.CompiledObjects.isSupportedLanguage(programmingLang) + } result <- Future { val start = System.currentTimeMillis() - val problems = scala.util.Try(code.api.dynamic.endpoint.helper.CompiledObjects.compileProblems(body.example_request_body, body.success_response_body, body.method_body)) match { + val problems = scala.util.Try(code.api.dynamic.endpoint.helper.CompiledObjects.compileProblems(body.example_request_body, body.success_response_body, body.method_body, programmingLang)) match { case scala.util.Success(ps) => ps case scala.util.Failure(e) => List(code.api.util.DynamicUtil.CompileProblem(0, 0, "ERROR", Option(e.getMessage).getOrElse(e.toString))) } val dependencyError: Option[String] = if (problems.nonEmpty) None - else scala.util.Try(code.api.dynamic.endpoint.helper.CompiledObjects(body.example_request_body, body.success_response_body, body.method_body).validateDependency()) match { + else scala.util.Try(code.api.dynamic.endpoint.helper.CompiledObjects(body.example_request_body, body.success_response_body, body.method_body, programmingLang).validateDependency()) match { case scala.util.Success(_) => None case scala.util.Failure(e: code.api.JsonResponseException) => Some(com.openbankproject.commons.util.JsonAliases.compactRender(e.jsonResponse.body)) case scala.util.Failure(e) => Some(Option(e.getMessage).getOrElse(e.toString)) @@ -6855,11 +6861,17 @@ object Http4s700 { |Send the fields that shape the compiled code: `request_verb`, `request_url`, the URL-encoded `method_body`, and the optional |`example_request_body` and `success_response_body` (they become the generated `RequestRootJsonClass` / `ResponseRootJsonClass`). | - |`errors` carry the compiler's messages with `line` and `column` relative to the method body you sent (the server's wrapper lines are + |`programming_lang` is the language of the method body, as on Create Dynamic Resource Doc: one of ${code.api.dynamic.endpoint.helper.CompiledObjects.supportedLanguagesText}. + |It is optional and defaults to Scala. Any other value is rejected with ${code.api.util.ErrorMessages.DynamicCodeLangNotSupport.takeWhile(_ != ':')}. + |A Java body is compiled as written, so `example_request_body` and `success_response_body` do not affect it. It must declare a public class + |implementing `Supplier>`; the function receives the raw request body, the path parameters and the CallContext. + | + |`errors` carry the compiler's messages with `line` and `column` relative to the method body you sent (the server's added lines are |subtracted; 0 when the compiler gave no position). When the body compiles and `dynamic_code_obp_calls_are_restricted` is on, |the dependency validator runs too and any forbidden call is reported in `dependency_error`. `compiles` is true only when both pass. | - |Nothing is evaluated or cached, but compiling is a full scalac run, so the same rules apply as for creating: the + |The compiler diagnostics are produced without running anything. When the body compiles, the dependency check builds it as Create would, + |so the same rules apply as for creating: the |`allow_user_generated_scala_code` kill switch, the create role, and at most $dynamicCompileCallsPerMinute calls per minute per user. | |Built for editors that let an author, or an assistant such as Opey, iterate on a body until it compiles before submitting it. @@ -6867,7 +6879,7 @@ object Http4s700 { |${userAuthenticationMessage(true)}""".stripMargin, JSONFactory700.dynamicResourceDocCompileJsonV700Example, JSONFactory700.dynamicCompileResultJsonV700Example, - List($AuthenticatedUserIsRequired, InvalidJsonFormat, UserHasMissingRoles, DynamicCodeExecutionDisabled, code.api.util.ErrorMessages.TooManyRequests, UnknownError), + List($AuthenticatedUserIsRequired, InvalidJsonFormat, UserHasMissingRoles, DynamicCodeExecutionDisabled, code.api.util.ErrorMessages.DynamicCodeLangNotSupport, code.api.util.ErrorMessages.TooManyRequests, UnknownError), apiTagDynamicResourceDoc :: apiTagDynamic :: Nil, Some(List(ApiRole.canCreateDynamicResourceDoc)), http4sPartialFunction = Some(compileDynamicResourceDoc) diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala index 2f8f25d4e6..80ca5fd32e 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala @@ -62,7 +62,7 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { // ─── Provenance for runtime-compiled dynamic code (v7.0.0 read-only exposure) ─── // The v4.0.0 create/update endpoints capture who created / last updated a piece of runtime - // code and a SHA-256 of its (decoded) method body into DB columns, but the v4 response shape + // code and a SHA-256 of its programming language and (decoded) method body into DB columns, but the v4 response shape // is frozen (STABLE) and does not carry them. These v7 GET endpoints expose that provenance, // wrapping the unchanged v4 resource JSON alongside a `provenance` object. case class ProvenanceJsonV700( @@ -1891,7 +1891,9 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { request_url: String, method_body: String, example_request_body: Option[JValue], - success_response_body: Option[JValue] + success_response_body: Option[JValue], + // "Scala" (the default when omitted) or "Java", as on Create Dynamic Resource Doc. + programming_lang: Option[String] = None ) case class DynamicCompileErrorJsonV700(line: Int, column: Int, severity: String, message: String) case class DynamicCompileResultJsonV700( @@ -1905,7 +1907,8 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { request_url = "/hello/world", method_body = java.net.URLEncoder.encode("Future.successful((Map(\"hello\" -> \"world\"), HttpCode.`200`(callContext)))", "UTF-8"), example_request_body = None, - success_response_body = Some(org.json4s.JsonAST.JObject(List(org.json4s.JsonAST.JField("hello", org.json4s.JsonAST.JString("world"))))) + success_response_body = Some(org.json4s.JsonAST.JObject(List(org.json4s.JsonAST.JField("hello", org.json4s.JsonAST.JString("world"))))), + programming_lang = Some("Scala") ) lazy val dynamicCompileResultJsonV700Example = DynamicCompileResultJsonV700( compiles = false, diff --git a/obp-api/src/main/scala/code/connectormethod/MappedConnectorMethodProvider.scala b/obp-api/src/main/scala/code/connectormethod/MappedConnectorMethodProvider.scala index 8bef23be08..6baae5852e 100644 --- a/obp-api/src/main/scala/code/connectormethod/MappedConnectorMethodProvider.scala +++ b/obp-api/src/main/scala/code/connectormethod/MappedConnectorMethodProvider.scala @@ -78,7 +78,7 @@ object MappedConnectorMethodProvider extends ConnectorMethodProvider { .Lang(entity.programmingLang) // provenance is set here from the authenticated user + computed hash, not from `entity` .CreatedByUserId(createdByUserId.getOrElse(null)) - .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) + .MethodBodyHash(APIUtil.dynamicCodeHash(entity.programmingLang, entity.decodedMethodBody)) .saveMe() }.map(ConnectorMethod.getJsonConnectorMethod) @@ -91,7 +91,7 @@ object MappedConnectorMethodProvider extends ConnectorMethodProvider { .Lang(programmingLang) // CreatedByUserId is left untouched; record who last changed the code + refresh the hash .UpdatedByUserId(updatedByUserId.getOrElse(null)) - .MethodBodyHash(APIUtil.sha256Hex(java.net.URLDecoder.decode(connectorMethodBody, "UTF-8"))) + .MethodBodyHash(APIUtil.dynamicCodeHash(programmingLang, java.net.URLDecoder.decode(connectorMethodBody, "UTF-8"))) .saveMe() }.map(ConnectorMethod.getJsonConnectorMethod) case _ => Empty diff --git a/obp-api/src/main/scala/code/dynamicMessageDoc/MappedDynamicMessageDocProvider.scala b/obp-api/src/main/scala/code/dynamicMessageDoc/MappedDynamicMessageDocProvider.scala index b23205999d..4cf20b8dda 100644 --- a/obp-api/src/main/scala/code/dynamicMessageDoc/MappedDynamicMessageDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicMessageDoc/MappedDynamicMessageDocProvider.scala @@ -99,7 +99,7 @@ object MappedDynamicMessageDocProvider extends DynamicMessageDocProvider { .Lang(entity.programmingLang) // provenance is set here from the authenticated user + computed hash, not from `entity` .CreatedByUserId(createdByUserId.getOrElse(null)) - .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) + .MethodBodyHash(APIUtil.dynamicCodeHash(entity.programmingLang, entity.decodedMethodBody)) .saveMe() }.map(DynamicMessageDoc.getJsonDynamicMessageDoc) } @@ -134,7 +134,7 @@ object MappedDynamicMessageDocProvider extends DynamicMessageDocProvider { .Lang(entity.programmingLang) // CreatedByUserId is left untouched; record who last changed the code + refresh the hash .UpdatedByUserId(updatedByUserId.getOrElse(null)) - .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) + .MethodBodyHash(APIUtil.dynamicCodeHash(entity.programmingLang, entity.decodedMethodBody)) .saveMe() }.map(DynamicMessageDoc.getJsonDynamicMessageDoc) case _ => Empty diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala index 63d68e88ab..3264b77e91 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDoc.scala @@ -52,6 +52,9 @@ class DynamicResourceDoc extends LongKeyedMapper[DynamicResourceDoc] with IdPK w object Tags extends MappedText(this) object Roles extends MappedText(this) object MethodBody extends MappedText(this) + // Source language of MethodBody: "Scala" (default) or "Java". Mirrors DynamicMessageDoc.Lang / + // ConnectorMethod.programmingLang — same field name/width convention, see DynamicEndpoints. + object Lang extends MappedString(this, 50) // Provenance: who created / last updated this runtime-compiled endpoint, and a SHA-256 of the // (decoded) method body so tampering / drift is detectable. Set server-side from the CallContext // user — never from the request body. createdAt / updatedAt come from the CreatedUpdated trait. @@ -85,7 +88,12 @@ object DynamicResourceDoc extends DynamicResourceDoc with LongKeyedMetaMapper[Dy successResponseBody = Option(dynamicResourceDoc.SuccessResponseBody.get).filter(StringUtils.isNotBlank).map(json.parse), errorResponseBodies = dynamicResourceDoc.ErrorResponseBodies.get, tags = dynamicResourceDoc.Tags.get, - roles = dynamicResourceDoc.Roles.get + roles = dynamicResourceDoc.Roles.get, + // Rows created before the Lang column existed have NULL there, not "Scala" -- a bare + // Lang.get would surface that as an empty/null programming_lang instead of falling back to + // JsonDynamicResourceDoc's own "Scala" default, since an explicit null argument bypasses a + // case class default (that only applies when the argument is omitted entirely). + programmingLang = Option(dynamicResourceDoc.Lang.get).filter(StringUtils.isNotBlank).getOrElse("Scala") ) } diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala index b11f4469ab..412363e261 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/DynamicResourceDocProvider.scala @@ -60,7 +60,12 @@ case class JsonDynamicResourceDoc( successResponseBody: Option[JValue], errorResponseBodies: String, tags: String, - roles: String + roles: String, + // Source language of methodBody: "Scala" (default) or "Java". Mirrors + // JsonConnectorMethod.programmingLang / JsonDynamicMessageDoc.programmingLang. Appended last + // (not inserted alphabetically) so existing named-arg call sites and JSON payloads that predate + // this field keep compiling/deserializing unchanged. + programmingLang: String = "Scala" ) extends JsonFieldReName { def decodedMethodBody: String = URLDecoder.decode(methodBody, "UTF-8") } diff --git a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala index 4364a7626a..ac1160efd5 100644 --- a/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala +++ b/obp-api/src/main/scala/code/dynamicResourceDoc/MappedDynamicResourceDocProvider.scala @@ -109,9 +109,10 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { .Tags(entity.tags) .Roles(entity.roles) .MethodBody(entity.methodBody) + .Lang(entity.programmingLang) // provenance is set here from the authenticated user + computed hash, not from `entity` .CreatedByUserId(createdByUserId.getOrElse(null)) - .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) + .MethodBodyHash(APIUtil.dynamicCodeHash(entity.programmingLang, entity.decodedMethodBody)) .saveMe() }.map(DynamicResourceDoc.getJsonDynamicResourceDoc) @@ -134,9 +135,10 @@ object MappedDynamicResourceDocProvider extends DynamicResourceDocProvider { .Tags(entity.tags) .Roles(entity.roles) .MethodBody(entity.methodBody) + .Lang(entity.programmingLang) // CreatedByUserId is left untouched; record who last changed the code + refresh the hash .UpdatedByUserId(updatedByUserId.getOrElse(null)) - .MethodBodyHash(APIUtil.sha256Hex(entity.decodedMethodBody)) + .MethodBodyHash(APIUtil.dynamicCodeHash(entity.programmingLang, entity.decodedMethodBody)) .saveMe() }.map(DynamicResourceDoc.getJsonDynamicResourceDoc) case _ => Empty diff --git a/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala b/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala index 0b8a694419..014308916b 100644 --- a/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala +++ b/obp-api/src/main/scala/code/dynamicchangerequest/MakerChecker.scala @@ -33,7 +33,7 @@ import java.util.Date import code.abacrule.{AbacRule, AbacRuleEngine, MappedAbacRuleProvider} import code.api.Constant import code.api.dynamic.endpoint.helper.CompiledObjects -import code.api.util.APIUtil.{getPropsAsBoolValue, getPropsAsIntValue, getPropsValue, sha256Hex} +import code.api.util.APIUtil.{dynamicCodeHash, getPropsAsBoolValue, getPropsAsIntValue, getPropsValue, sha256Hex} import code.api.util.DynamicUtil.Validation import code.api.util.{CallContext, ErrorMessages} import code.api.v6_0_0.{CreateAbacRuleJsonV600, UpdateAbacRuleJsonV600} @@ -111,15 +111,21 @@ object MakerChecker extends MdcLoggable { private def blank(s: String): Boolean = StringUtils.isBlank(s) - private def bodyHashOf(storedHash: String, encodedBody: String): String = - if (!blank(storedHash)) storedHash - else sha256Hex(URLDecoder.decode(Option(encodedBody).getOrElse(""), "UTF-8")) + /** + * The hash of a code row as it is now: its language and its decoded body (see APIUtil.dynamicCodeHash). + * + * Always recomputed from the row, never read from the stored MethodBodyHash column. The guard + * compares this with ApprovedHash, and a stored hash would let a direct database edit of the body + * or the language keep the approval of the code it replaced. + */ + private def bodyHashOf(programmingLang: String, encodedBody: String): String = + dynamicCodeHash(programmingLang, URLDecoder.decode(Option(encodedBody).getOrElse(""), "UTF-8")) /** The live target's body hash, Empty when the target does not exist. */ def currentBodyHash(targetType: DynamicChangeRequestTargetType, targetId: String): Box[String] = targetType match { - case DYNAMIC_RESOURCE_DOC => DynamicResourceDoc.find(By(DynamicResourceDoc.DynamicResourceDocId, targetId)).map(r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get)) - case DYNAMIC_MESSAGE_DOC => DynamicMessageDoc.find(By(DynamicMessageDoc.DynamicMessageDocId, targetId)).map(r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get)) - case CONNECTOR_METHOD => ConnectorMethod.find(By(ConnectorMethod.ConnectorMethodId, targetId)).map(r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get)) + case DYNAMIC_RESOURCE_DOC => DynamicResourceDoc.find(By(DynamicResourceDoc.DynamicResourceDocId, targetId)).map(r => bodyHashOf(r.Lang.get, r.MethodBody.get)) + case DYNAMIC_MESSAGE_DOC => DynamicMessageDoc.find(By(DynamicMessageDoc.DynamicMessageDocId, targetId)).map(r => bodyHashOf(r.Lang.get, r.MethodBody.get)) + case CONNECTOR_METHOD => ConnectorMethod.find(By(ConnectorMethod.ConnectorMethodId, targetId)).map(r => bodyHashOf(r.Lang.get, r.MethodBody.get)) case ABAC_RULE => AbacRule.find(By(AbacRule.AbacRuleId, targetId)).map(r => sha256Hex(Option(r.RuleCode.get).getOrElse(""))) case _ => Empty } @@ -138,18 +144,18 @@ object MakerChecker extends MdcLoggable { def isExecutableDynamicResourceDoc(dynamicResourceDocId: String): Boolean = DynamicResourceDoc.find(By(DynamicResourceDoc.DynamicResourceDocId, dynamicResourceDocId)) - .map(r => executable(r.IsActive.get, bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), r.ApprovedHash.get, DYNAMIC_RESOURCE_DOC)) + .map(r => executable(r.IsActive.get, bodyHashOf(r.Lang.get, r.MethodBody.get), r.ApprovedHash.get, DYNAMIC_RESOURCE_DOC)) .getOrElse(false) def isExecutableDynamicMessageDoc(dynamicMessageDocId: String): Boolean = memoGuard("dmd_" + dynamicMessageDocId) { DynamicMessageDoc.find(By(DynamicMessageDoc.DynamicMessageDocId, dynamicMessageDocId)) - .map(r => executable(r.IsActive.get, bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), r.ApprovedHash.get, DYNAMIC_MESSAGE_DOC)) + .map(r => executable(r.IsActive.get, bodyHashOf(r.Lang.get, r.MethodBody.get), r.ApprovedHash.get, DYNAMIC_MESSAGE_DOC)) .getOrElse(false) } def isExecutableConnectorMethod(connectorMethodId: String): Boolean = memoGuard("cm_" + connectorMethodId) { ConnectorMethod.find(By(ConnectorMethod.ConnectorMethodId, connectorMethodId)) - .map(r => executable(r.IsActive.get, bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), r.ApprovedHash.get, CONNECTOR_METHOD)) + .map(r => executable(r.IsActive.get, bodyHashOf(r.Lang.get, r.MethodBody.get), r.ApprovedHash.get, CONNECTOR_METHOD)) .getOrElse(false) } @@ -359,7 +365,7 @@ object MakerChecker extends MdcLoggable { for { body <- parseAs[JsonDynamicResourceDoc](request.proposedPayload) _ <- compileBox("dynamic resource doc") { - val compiled = CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody) + val compiled = CompiledObjects(body.exampleRequestBody, body.successResponseBody, body.methodBody, body.programmingLang) compiled.validateDependency() Full(compiled) } @@ -465,17 +471,17 @@ object MakerChecker extends MdcLoggable { targetType match { case DYNAMIC_RESOURCE_DOC => DynamicResourceDoc.find(By(DynamicResourceDoc.DynamicResourceDocId, targetId)).map { r => - val h = bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get) + val h = bodyHashOf(r.Lang.get, r.MethodBody.get) r.MethodBodyHash(h).ApprovedHash(h).IsActive(true).save; () } case DYNAMIC_MESSAGE_DOC => DynamicMessageDoc.find(By(DynamicMessageDoc.DynamicMessageDocId, targetId)).map { r => - val h = bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get) + val h = bodyHashOf(r.Lang.get, r.MethodBody.get) r.MethodBodyHash(h).ApprovedHash(h).IsActive(true).save; () } case CONNECTOR_METHOD => ConnectorMethod.find(By(ConnectorMethod.ConnectorMethodId, targetId)).map { r => - val h = bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get) + val h = bodyHashOf(r.Lang.get, r.MethodBody.get) r.MethodBodyHash(h).ApprovedHash(h).IsActive(true).save; () } case ABAC_RULE => @@ -500,6 +506,89 @@ object MakerChecker extends MdcLoggable { // ─── boot ────────────────────────────────────────────────────────────────── + /** MigrationScriptLog entry that records the one-off rehash below. */ + val rehashMigrationName = "rehashDynamicCodeWithLanguage" + + /** + * This moves stored code hashes from the old form (SHA-256 of the body alone) to the current form + * (SHA-256 of the language and the body, see APIUtil.dynamicCodeHash). It runs once per database, + * at every boot until it has succeeded, whether or not maker/checker is enabled, because + * MethodBodyHash is also reported as provenance. + * + * It runs from Boot rather than as a Migration.database script because those run only on instances + * that switch migration scripts on. Without this step, every approved row on any other instance + * would stop executing at the upgrade, since its ApprovedHash would no longer equal the hash the + * guard computes. + * + * For each Dynamic Resource Doc, Dynamic Message Doc and Connector Method: + * - MethodBodyHash is recomputed in the new form; + * - ApprovedHash is moved to the new form only when it equals the old-form hash of the row's + * current body, so the approval is carried over exactly where it was still valid. A row whose + * body no longer matches its approval stays unexecutable, as it was before; + * - a pending change request whose CurrentPayloadHash equals the old-form hash of its target is + * moved too, so it does not turn stale merely because of this rehash. + * + * What it cannot do: an approval granted before this change bound the body only, so a row whose + * language was changed after approval (through the API, without a new body) keeps that approval, + * now bound to its current language. From this change on, changing the language requires approval. + * + * Rerunning it is harmless: once moved, an ApprovedHash no longer equals the old form and is left + * alone, so several nodes booting at once cannot do damage. + */ + def rehashDynamicCodeWithLanguage(): Unit = { + val logProvider = code.migration.MigrationScriptLogProvider.migrationScriptLogProvider.vend + if (!logProvider.isExecuted(rehashMigrationName)) { + val start = System.currentTimeMillis() + // targetType -> (old-form hash -> new-form hash) of every row, for moving pending change requests. + val moved = scala.collection.mutable.Map[(String, String), (String, String)]() + def rehash[T](targetType: DynamicChangeRequestTargetType, rows: List[T])( + id: T => String, lang: T => String, encodedBody: T => String, approvedHash: T => String, write: (T, String, Option[String]) => Unit + ): String = { + var approvalsMoved = 0 + rows.foreach { r => + val decoded = URLDecoder.decode(Option(encodedBody(r)).getOrElse(""), "UTF-8") + val oldHash = sha256Hex(decoded) + val newHash = dynamicCodeHash(lang(r), decoded) + val approvalStillValid = approvedHash(r) == oldHash + if (approvalStillValid) approvalsMoved += 1 + write(r, newHash, if (approvalStillValid) Some(newHash) else None) + moved((targetType.toString, id(r))) = (oldHash, newHash) + } + s"$targetType: ${rows.size} rows, $approvalsMoved approvals moved" + } + tryo { + val summary = List( + rehash(DYNAMIC_RESOURCE_DOC, DynamicResourceDoc.findAll())( + _.DynamicResourceDocId.get, _.Lang.get, _.MethodBody.get, _.ApprovedHash.get, + (r, h, approved) => { r.MethodBodyHash(h); approved.foreach(r.ApprovedHash(_)); r.save; () }), + rehash(DYNAMIC_MESSAGE_DOC, DynamicMessageDoc.findAll())( + _.DynamicMessageDocId.get, _.Lang.get, _.MethodBody.get, _.ApprovedHash.get, + (r, h, approved) => { r.MethodBodyHash(h); approved.foreach(r.ApprovedHash(_)); r.save; () }), + rehash(CONNECTOR_METHOD, ConnectorMethod.findAll())( + _.ConnectorMethodId.get, _.Lang.get, _.MethodBody.get, _.ApprovedHash.get, + (r, h, approved) => { r.MethodBodyHash(h); approved.foreach(r.ApprovedHash(_)); r.save; () }) + ) + val pending = DynamicChangeRequest.findAll(By(DynamicChangeRequest.Status, DynamicChangeRequestStatus.INITIATED.toString)) + var requestsMoved = 0 + pending.foreach { request => + moved.get((request.TargetType.get, request.TargetId.get)).foreach { case (oldHash, newHash) => + if (request.CurrentPayloadHash.get == oldHash) { request.CurrentPayloadHash(newHash).save; requestsMoved += 1 } + } + } + (summary :+ s"pending change requests moved: $requestsMoved").mkString(", ") + } match { + case Full(summary) => + val comment = s"Code hashes now cover the programming language as well as the body ($summary)" + logger.warn(s"rehashDynamicCodeWithLanguage says: $comment") + logProvider.saveLog(rehashMigrationName, code.api.util.APIUtil.gitCommit, true, start, System.currentTimeMillis(), comment) + case f: Failure => + logger.error(s"rehashDynamicCodeWithLanguage says: failed, will retry at next boot: ${f.messageChain}") + logProvider.saveLog(rehashMigrationName, code.api.util.APIUtil.gitCommit, false, start, System.currentTimeMillis(), f.messageChain) + case _ => () + } + } + } + /** MigrationScriptLog entry that records the one-off seeding below; the seed never runs twice on a database. */ val seedMigrationName = "seedDynamicCodeApprovedHashes" @@ -525,11 +614,11 @@ object MakerChecker extends MdcLoggable { tryo { List( seed("DynamicResourceDoc", DynamicResourceDoc.findAll().filter(r => blank(r.ApprovedHash.get)))( - r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), + r => bodyHashOf(r.Lang.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), seed("DynamicMessageDoc", DynamicMessageDoc.findAll().filter(r => blank(r.ApprovedHash.get)))( - r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), + r => bodyHashOf(r.Lang.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), seed("ConnectorMethod", ConnectorMethod.findAll().filter(r => blank(r.ApprovedHash.get)))( - r => bodyHashOf(r.MethodBodyHash.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), + r => bodyHashOf(r.Lang.get, r.MethodBody.get), (r, h) => { r.MethodBodyHash(h).ApprovedHash(h).save; () }), seed("AbacRule", AbacRule.findAll().filter(r => blank(r.ApprovedHash.get)))( r => sha256Hex(Option(r.RuleCode.get).getOrElse("")), (r, h) => { r.ApprovedHash(h).save; () }) ).mkString(", ") diff --git a/obp-api/src/test/resources/frozen_type_meta_data b/obp-api/src/test/resources/frozen_type_meta_data index 6d3596e485..924862c8b8 100644 Binary files a/obp-api/src/test/resources/frozen_type_meta_data and b/obp-api/src/test/resources/frozen_type_meta_data differ diff --git a/obp-api/src/test/resources/frozen_type_meta_data.txt b/obp-api/src/test/resources/frozen_type_meta_data.txt index 100da9936f..b62b9d93c0 100644 --- a/obp-api/src/test/resources/frozen_type_meta_data.txt +++ b/obp-api/src/test/resources/frozen_type_meta_data.txt @@ -3671,6 +3671,7 @@ field code.dynamicResourceDoc.JsonDynamicResourceDoc errorResponseBodies String field code.dynamicResourceDoc.JsonDynamicResourceDoc exampleRequestBody Option[org.json4s.JValue] field code.dynamicResourceDoc.JsonDynamicResourceDoc methodBody String field code.dynamicResourceDoc.JsonDynamicResourceDoc partialFunctionName String +field code.dynamicResourceDoc.JsonDynamicResourceDoc programmingLang String field code.dynamicResourceDoc.JsonDynamicResourceDoc requestUrl String field code.dynamicResourceDoc.JsonDynamicResourceDoc requestVerb String field code.dynamicResourceDoc.JsonDynamicResourceDoc roles String diff --git a/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala b/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala new file mode 100644 index 0000000000..ab984a317b --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/DynamicUtilJavaHttp4sEndpointTest.scala @@ -0,0 +1,109 @@ +package code.api.util + +import cats.effect.IO +import cats.effect.unsafe.implicits.global +import org.http4s.{Method, Request, Uri} +import org.json4s.native.JsonMethods.parse +import org.scalatest.{FeatureSpec, GivenWhenThen, Matchers} + +/** + * Focused unit test for DynamicUtil.createJavaHttp4sEndpoint, isolated from the full + * register -> role-check -> HTTP-dispatch round trip (see DynamicResourceDocJavaTest for that). + * Exercises the adapter directly: compiled Java Supplier> -> + * Http4sEndpointIO.apply(Request[IO]) -> CallContext => IO[Response[IO]]. + */ +class DynamicUtilJavaHttp4sEndpointTest extends FeatureSpec with Matchers with GivenWhenThen { + + private val echoMethodBody = + """package code.api.util.dynamic; + | + |import code.api.util.CallContext; + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaHttp4sEndpointUnitTest implements Supplier> { + | private Object apply(Object[] args) { + | String rawBody = (String) args[0]; + | @SuppressWarnings("unchecked") + | Map pathParams = (Map) args[1]; + | CallContext cc = (CallContext) args[2]; + | + | Map response = new LinkedHashMap<>(); + | response.put("echoed_body", rawBody); + | response.put("path_param_count", pathParams.size()); + | response.put("correlation_id", cc.correlationId()); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + feature("DynamicUtil.createJavaHttp4sEndpoint compiles a Java method_body into a native Http4sEndpointIO") { + + scenario("the compiled endpoint reads args(0)/args(1)/args(2) and serves 200 JSON") { + Given("a Java method_body compiled via createJavaHttp4sEndpoint") + val endpoint = DynamicUtil.createJavaHttp4sEndpoint(echoMethodBody).openOrThrowException("compilation failed") + + When("the compiled endpoint handles a request carrying a body, no path params, and a CallContext") + val req = Request[IO](method = Method.POST, uri = Uri.unsafeFromString("/test")) + val cc = CallContext(httpBody = Some("""{"hello":"world"}"""), correlationId = "test-correlation-id") + val resp = endpoint.apply(req)(cc).unsafeRunSync() + + Then("the response is 200 and echoes the body, the (empty) path params, and the CallContext's correlationId") + resp.status.code should equal(200) + val bodyString = resp.body.through(fs2.text.utf8.decode).compile.string.unsafeRunSync() + val json = parse(bodyString) + (json \ "echoed_body").values should equal("""{"hello":"world"}""") + (json \ "path_param_count").values should equal(BigInt(0)) + (json \ "correlation_id").values should equal("test-correlation-id") + } + + scenario("a Java compile error is reported as a Box Failure, not a thrown exception") { + Given("a method_body that is not valid Java") + val badMethodBody = "this is not valid java at all" + + When("we try to compile it") + val result = DynamicUtil.createJavaHttp4sEndpoint(badMethodBody) + + Then("compilation fails gracefully") + result.isDefined should equal(false) + } + + scenario("a Java method_body that throws at runtime is recovered as a 500, not an uncaught exception") { + Given("a Java method_body whose apply() throws") + val throwingMethodBody = + """package code.api.util.dynamic; + | + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaHttp4sEndpointThrowingTest implements Supplier> { + | private Object apply(Object[] args) { + | throw new RuntimeException("boom"); + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + val endpoint = DynamicUtil.createJavaHttp4sEndpoint(throwingMethodBody).openOrThrowException("compilation failed") + + When("the compiled endpoint is invoked") + val req = Request[IO](method = Method.POST, uri = Uri.unsafeFromString("/test")) + val resp = endpoint.apply(req)(CallContext()).unsafeRunSync() + + Then("the response is 500 rather than the IO failing") + resp.status.code should equal(500) + val bodyString = resp.body.through(fs2.text.utf8.decode).compile.string.unsafeRunSync() + bodyString should include("boom") + } + } +} diff --git a/obp-api/src/test/scala/code/api/v4_0_0/ConnectorMethodTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/ConnectorMethodTest.scala index 666ff9b0e3..861c261865 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/ConnectorMethodTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/ConnectorMethodTest.scala @@ -104,7 +104,7 @@ class ConnectorMethodTest extends V400ServerSetup { .find(net.liftweb.mapper.By(code.connectormethod.ConnectorMethod.ConnectorMethodId, connectorMethod.connectorMethodId.getOrElse(""))) .openOrThrowException("stored connector method not found") storedConnectorMethod.CreatedByUserId.get should be (resourceUser1.userId) - storedConnectorMethod.MethodBodyHash.get should be (code.api.util.APIUtil.sha256Hex(postConnectorMethod.decodedMethodBody)) + storedConnectorMethod.MethodBodyHash.get should be (code.api.util.APIUtil.dynamicCodeHash(postConnectorMethod.programmingLang, postConnectorMethod.decodedMethodBody)) Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanCreateMethodRouting.toString) diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicMessageDocTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicMessageDocTest.scala index cc4265ebef..558d0dbe3a 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicMessageDocTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicMessageDocTest.scala @@ -102,7 +102,7 @@ class DynamicMessageDocTest extends V400ServerSetup { .find(net.liftweb.mapper.By(code.dynamicMessageDoc.DynamicMessageDoc.DynamicMessageDocId, dynamicMessageDoc.dynamicMessageDocId.getOrElse(""))) .openOrThrowException("stored dynamic message doc not found") storedMessageDoc.CreatedByUserId.get should be (resourceUser1.userId) - storedMessageDoc.MethodBodyHash.get should be (code.api.util.APIUtil.sha256Hex(postDynamicMessageDoc.decodedMethodBody)) + storedMessageDoc.MethodBodyHash.get should be (code.api.util.APIUtil.dynamicCodeHash(postDynamicMessageDoc.programmingLang, postDynamicMessageDoc.decodedMethodBody)) Then(s"we test the $ApiEndpoint2") diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala new file mode 100644 index 0000000000..aaad696ae8 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaSecurityValidationTest.scala @@ -0,0 +1,174 @@ +package code.api.v4_0_0 + +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.api.util.ErrorMessages.DynamicResourceDocMethodDependency +import code.entitlement.Entitlement +import com.openbankproject.commons.model.ErrorMessage +import org.json4s.native.Serialization.write + +/** + * With dynamic_code_obp_calls_are_restricted=true, a Java method_body that calls an OBP method NOT + * on the dependency whitelist must be rejected -- proving createJavaHttp4sEndpoint validates the + * real compiled Java class (getCompiledInstance), not just its Scala wrapper. + */ +class DynamicResourceDocJavaSecurityValidationTest extends V400ServerSetup { + + private def maliciousMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaSecurityProbe implements Supplier> { + | private Object apply(Object[] args) { + | // APIUtil.getPropsValue is NOT on dynamic_code_allowed_obp_methods' + | // whitelist (only errorJsonResponse*/scalaFutureToLaFuture/futureToBoxedResponse are). + | String secret = code.api.util.APIUtil$.MODULE$.getPropsValue("hostname", "none"); + | Map response = new LinkedHashMap<>(); + | response.put("leaked", secret); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + // Mirrors sample.props.template's default dynamic_code_allowed_obp_methods exactly, + // minus the trailing newlines/line-continuations -- deliberately does NOT list APIUtil.getPropsValue. + private def defaultDependenciesWhitelist: String = + """[NewStyle.function.getClass.getTypeName -> "*", CompiledObjects.getClass.getTypeName -> "sandbox", HttpCode.getClass.getTypeName -> "200", DynamicCompileEndpoint.getClass.getTypeName -> "getPathParams, scalaFutureToBoxedJsonResponse", APIUtil.getClass.getTypeName -> "errorJsonResponse, errorJsonResponse$default$1, errorJsonResponse$default$2, errorJsonResponse$default$3, errorJsonResponse$default$4, scalaFutureToLaFuture, futureToBoxedResponse", ErrorMessages.getClass.getTypeName -> "*", ExecutionContext.Implicits.getClass.getTypeName -> "global", JSONFactory400.getClass.getTypeName -> "createBanksJson", classOf[CallContext].getTypeName -> "*", classOf[ResourceDoc].getTypeName -> "getPathParams", "scala.reflect.runtime.package$" -> "universe", PractiseEndpoint.getClass.getTypeName + "*" -> "*"]""" + + // Deliberately does NOT set show_used_connector_methods: that prop exists to opt in to an + // unrelated, expensive introspection/reporting feature and was never meant to gate security + // validation, which happens to reuse the same underlying bytecode scan. An operator who reads + // only dynamic_code_obp_calls_are_restricted's own prop documentation and sets just these two + // props (as this method does) must still get real enforcement -- proving that is the point of + // every scenario below. + // + // Block body (not `= setPropsValues(...)`) so .github/scripts/check_test_isolation.py's brace + // scanner sees an opening `{` right after `def enableStrictValidation` and treats this as a + // safe "helper called from scenarios" scope rather than a class-body-level setPropsValues call. + private def enableStrictValidation(): Unit = { + setPropsValues( + "dynamic_code_obp_calls_are_restricted" -> "true", + "dynamic_code_allowed_obp_methods" -> defaultDependenciesWhitelist + ) + } + + // Every Java method_body implements Supplier> per convention (see + // DynamicUtil.createJavaHttp4sEndpoint's doc comment). javac always erases that generic + // Supplier.get() to a synthetic bridge method `Object get()` whose body just invokevirtual-calls + // the real, properly-typed get() -- an ordinary same-class call regardless of what the body does. + private def benignMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaSecurityBenignProbe implements Supplier> { + | private Object apply(Object[] args) { + | Map response = new LinkedHashMap<>(); + | response.put("greeting", "hello"); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + private def createRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + + feature("Security validation of Java method_body against dynamic_code_allowed_obp_methods") { + + // Regression guard: the Supplier.get() generics-erasure bridge method's same-class call to the + // real get() must not itself be treated as a call to a forbidden method. Without this, every + // Java doc -- malicious or not -- was rejected under strict validation, because the compiled + // class lives under the OBP-owned code.* package but its randomly-generated name can never + // appear in a static whitelist. + scenario("Registering a benign Java doc succeeds even with strict validation enabled") { + enableStrictValidation() + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "benignProbeTest", + requestUrl = "/benign_probe_test/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(benignMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val resp = makePostRequest(createRequest, write(doc)) + + Then("the compile succeeds -- the Supplier.get() bridge method's self-call is not a forbidden dependency") + resp.code should equal(201) + } + scenario("Registering a Java doc that calls a non-whitelisted OBP method is rejected with 400") { + enableStrictValidation() + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "securityProbeTest", + requestUrl = "/security_probe_test/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(maliciousMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val resp = makePostRequest(createRequest, write(doc)) + + Then("the compile is rejected with 400 DynamicResourceDocMethodDependency, not accepted") + resp.code should equal(400) + resp.body.extract[ErrorMessage].message should include(DynamicResourceDocMethodDependency) + } + + // Regression guard for the bug createJavaHttp4sEndpoint had before it split compilation from + // validation: memoJavaCompiledScript (formerly memoJavaHttp4sEndpoint) memoized the WHOLE + // Box[Http4sEndpointIO], keyed only by the exact method_body string. A doc compiled once while + // validation was off got a cached Full(...) that a later, identical create call -- made AFTER + // validation was turned on and the whitelist tightened -- would silently reuse, never + // re-running Validation.validateDependency at all. This scenario reproduces exactly that + // sequence: compile the same malicious source once with validation off (succeeds, populates + // the compile cache), then enable strict validation and resubmit the identical source. + scenario("A Java source compiled once while validation was off is still validated on a later create call") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val firstDoc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "cacheBypassProbeTest1", + requestUrl = "/cache_bypass_probe_test_1/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(maliciousMethodBody, "UTF-8"), + programmingLang = "Java" + ) + When("validation is off (the suite default) and we compile the malicious source for the first time") + val firstResp = makePostRequest(createRequest, write(firstDoc)) + firstResp.code should equal(201) + + When("validation is then turned on with the same source resubmitted under a different doc") + enableStrictValidation() + val secondDoc = firstDoc.copy( + partialFunctionName = "cacheBypassProbeTest2", + requestUrl = "/cache_bypass_probe_test_2/MY_USER_ID" + ) + val secondResp = makePostRequest(createRequest, write(secondDoc)) + + Then("the second create is still rejected -- the compile-result cache must not bypass fresh validation") + secondResp.code should equal(400) + secondResp.body.extract[ErrorMessage].message should include(DynamicResourceDocMethodDependency) + } + } +} diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala new file mode 100644 index 0000000000..aa12c85bc8 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocJavaTest.scala @@ -0,0 +1,141 @@ +package code.api.v4_0_0 + +import org.json4s._ +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.api.util.ErrorMessages.DynamicCodeLangNotSupport +import code.dynamicResourceDoc.JsonDynamicResourceDoc +import code.entitlement.Entitlement +import com.openbankproject.commons.model.ErrorMessage +import com.openbankproject.commons.util.json +import org.json4s.native.JsonMethods.{compact, parse => parseJson, render} +import org.json4s.native.Serialization.write + +/** + * Java-language coverage for the DynamicResourceDoc runtime-compilation mechanism. + * DynamicResourceDocTest.scala covers the (unchanged) Scala-language path end-to-end; these + * scenarios exercise the new `programming_lang = "Java"` dispatch added to + * DynamicEndpoints.CompiledObjects / DynamicUtil.createJavaHttp4sEndpoint, plus the + * backward-compat and unsupported-language guards added alongside it. + */ +class DynamicResourceDocJavaTest extends V400ServerSetup { + + private def createDynamicResourceDocsRequest = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + + // Java-side convention: the pasted class implements Supplier>. + // args(0) = raw request body (String, or null), args(1) = path params (java.util.Map), + // args(2) = the CallContext. See DynamicUtil.createJavaHttp4sEndpoint's doc comment. + private def javaRoleTestMethodBody: String = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class DynamicJavaResourceDocRoleTest implements Supplier> { + | private Object apply(Object[] args) { + | String rawBody = (String) args[0]; + | @SuppressWarnings("unchecked") + | Map pathParams = (Map) args[1]; + | String myUserId = pathParams.get("MY_USER_ID"); + | + | Map response = new LinkedHashMap<>(); + | response.put("user_id_from_path", myUserId + "_from_path"); + | response.put("received_body", rawBody); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + feature("Native execution of a runtime-compiled dynamic resource doc with a Java method_body") { + + scenario("Create a role-gated Java-language dynamic resource doc and verify 401 / 403 / 200") { + val dynamicRole = "CanCallJavaPieceCRoleTest" + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a Java-language dynamic resource doc gated by that role") + val createReq = createDynamicResourceDocsRequest + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = dynamicRole, + partialFunctionName = "javaPieceCRoleTest", + requestUrl = "/my_java_role_user/MY_USER_ID", + methodBody = java.net.URLEncoder.encode(javaRoleTestMethodBody, "UTF-8"), + programmingLang = "Java" + ) + val createResp = makePostRequest(createReq, write(doc)) + Then("We should get a 201") + createResp.code should equal(201) + createResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Java") + + val callUrl = dynamicEndpoint_Request / "dynamic-resource-doc" / "my_java_role_user" / "user-1" + val body = """{"name":"Jhon","age":12,"hobby":["coding"]}""" + + assertRoleGated401Then403Then200(callUrl, body, dynamicRole) { resp200 => + val rendered = json.compactRender(resp200.body) + rendered should include("user-1_from_path") + rendered should include("Jhon") + } + } + + scenario("Reject an unsupported programming_lang before attempting compilation") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a dynamic resource doc with an unsupported programming_lang") + val createReq = createDynamicResourceDocsRequest + val doc = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + partialFunctionName = "unsupportedLangTest", + requestUrl = "/unsupported_lang_test/MY_USER_ID", + programmingLang = "Python" + ) + val resp = makePostRequest(createReq, write(doc)) + + Then("We should get a 400 DynamicCodeLangNotSupport, not a compile-failure error") + resp.code should equal(400) + resp.body.extract[ErrorMessage].message should include(DynamicCodeLangNotSupport) + } + + scenario("Backward compatibility: a request body with programming_lang entirely omitted still creates a Scala-language doc") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + When("We create a dynamic resource doc from a JSON payload that predates the programming_lang field") + val posted = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + roles = "", + partialFunctionName = "preExistingClientTest", + requestUrl = "/pre_existing_client_test/MY_USER_ID" + ) + // Simulate an old client payload: strip programming_lang out entirely rather than relying on + // Serialization.write, which always emits every case-class field (default or not). + val fullJson = parseJson(write(posted)) + val withoutLang = fullJson.removeField { case (name, _) => name == "programming_lang" } + val requestBodyStr = compact(render(withoutLang)) + requestBodyStr should not include "programming_lang" + + val createReq = createDynamicResourceDocsRequest + val createResp = makePostRequest(createReq, requestBodyStr) + + Then("We should get a 201 and the stored/served doc defaults to the Scala language") + createResp.code should equal(201) + createResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Scala") + + Then("calling the endpoint still compiles and serves via the (unchanged) Scala template path") + val callReq = (dynamicEndpoint_Request / "dynamic-resource-doc" / "pre_existing_client_test" / "user-1").POST <@ (user1) + val callResp = makePostRequest(callReq, """{"name":"Jhon","age":12,"hobby":["coding"]}""") + callResp.code should equal(200) + val rendered = json.compactRender(callResp.body) + rendered should include("user-1_from_path") + rendered should include("Jhon") + } + } +} diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala index 543f58b2a8..0210a3a5ce 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicResourceDocTest.scala @@ -29,7 +29,7 @@ import org.json4s._ import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON import code.api.util.APIUtil.OAuth._ import code.api.util.ApiRole._ -import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, DynamicResourceDocAlreadyExists, DynamicResourceDocNotFound, UserHasMissingRoles} +import code.api.util.ErrorMessages.{DynamicResourceDocAlreadyExists, DynamicResourceDocNotFound, UserHasMissingRoles} import code.api.util.ApiRole import code.api.v4_0_0.Http4s400.Implementations4_0_0 import code.dynamicResourceDoc.JsonDynamicResourceDoc @@ -315,21 +315,9 @@ class DynamicResourceDocTest extends V400ServerSetup { val callUrl = dynamicEndpoint_Request / "dynamic-resource-doc" / "my_role_user" / "user-1" val body = """{"name":"Jhon","age":12,"hobby":["coding"]}""" - Then("calling without authentication returns 401") - val resp401 = makePostRequest(callUrl.POST, body) - resp401.code should equal(401) - resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) - - Then("calling authenticated but without the role returns 403") - val resp403 = makePostRequest(callUrl.POST <@ (user1), body) - resp403.code should equal(403) - resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) - - Then("granting the role makes the call succeed (200)") - Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) - val resp200 = makePostRequest(callUrl.POST <@ (user1), body) - resp200.code should equal(200) - json.compactRender(resp200.body) should include("_from_path") + assertRoleGated401Then403Then200(callUrl, body, dynamicRole) { resp200 => + json.compactRender(resp200.body) should include("_from_path") + } } // Regression guard for DynamicEndpointCodeGenerator.buildTemplate: the template served by @@ -411,7 +399,7 @@ class DynamicResourceDocTest extends V400ServerSetup { .find(net.liftweb.mapper.By(code.dynamicResourceDoc.DynamicResourceDoc.DynamicResourceDocId, docId)) .openOrThrowException("stored dynamic resource doc not found") storedRow.CreatedByUserId.get should be(resourceUser1.userId) - storedRow.MethodBodyHash.get should be(code.api.util.APIUtil.sha256Hex(posted.decodedMethodBody)) + storedRow.MethodBodyHash.get should be(code.api.util.APIUtil.dynamicCodeHash(posted.programmingLang, posted.decodedMethodBody)) When("We update the doc with a changed method body") val changedMethodBody = URLEncoder.encode( @@ -424,7 +412,43 @@ class DynamicResourceDocTest extends V400ServerSetup { Then("created_by_user_id is preserved, updated_by_user_id is recorded, and the hash reflects the new body") storedRow.CreatedByUserId.get should be(resourceUser1.userId) storedRow.UpdatedByUserId.get should be(resourceUser1.userId) - storedRow.MethodBodyHash.get should be(code.api.util.APIUtil.sha256Hex(URLDecoder.decode(changedMethodBody, "UTF-8"))) + storedRow.MethodBodyHash.get should be(code.api.util.APIUtil.dynamicCodeHash(posted.programmingLang, URLDecoder.decode(changedMethodBody, "UTF-8"))) + } + + // Regression guard: rows created before the Lang column existed have a genuine SQL NULL there + // (Schemifier's ALTER TABLE ADD COLUMN sets no default), not "Scala". An explicit null argument + // bypasses JsonDynamicResourceDoc's own programmingLang="Scala" default -- that default only + // applies when the argument is omitted entirely -- so a bare Lang.get would have surfaced as a + // null/empty programming_lang in the API response instead of falling back to "Scala". + scenario("A dynamic resource doc row predating the programming_lang column still reports \"Scala\"", ApiEndpoint1, ApiEndpoint3, VersionOfApi) { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canGetDynamicResourceDoc.toString) + + When("We create a dynamic resource doc") + val createReq = (v4_0_0_Request / "management" / "dynamic-resource-docs").POST <@ (user1) + val posted = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + bankId = None, + partialFunctionName = "preDatesLangColumnTest", + requestUrl = "/pre_dates_lang_column_test/MY_USER_ID" + ) + val createResp = makePostRequest(createReq, write(posted)) + createResp.code should equal(201) + val docId = (createResp.body \ "dynamic_resource_doc_id").values.toString + + When("its lang column is forced to a genuine SQL NULL, bypassing the ORM (which always writes \"Scala\")") + import code.dynamicResourceDoc.DynamicResourceDoc + net.liftweb.mapper.DB.runUpdate( + s"UPDATE ${DynamicResourceDoc.dbTableName} SET ${DynamicResourceDoc.Lang.dbColumnName} = NULL " + + s"WHERE ${DynamicResourceDoc.DynamicResourceDocId.dbColumnName} = ?", + List(docId) + ) + + Then("GET still reports programming_lang as \"Scala\", not null or empty") + val getReq = (v4_0_0_Request / "management" / "dynamic-resource-docs" / docId).GET <@ (user1) + val getResp = makeGetRequest(getReq) + getResp.code should equal(200) + getResp.body.extract[JsonDynamicResourceDoc].programmingLang should equal("Scala") } } diff --git a/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala b/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala index 114d59786e..4f97200a61 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/V400ServerSetup.scala @@ -48,9 +48,10 @@ import code.metadata.comments.MappedComment import code.metadata.narrative.MappedNarrative import code.metadata.transactionimages.MappedTransactionImage import code.metadata.wheretags.MappedWhereTag +import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, UserHasMissingRoles} import code.setup.{APIResponse, DefaultUsers, ServerSetupWithTestData} import code.transactionattribute.MappedTransactionAttribute -import com.openbankproject.commons.model.{AccountId, AccountRoutingJsonV121, AmountOfMoneyJsonV121, BankId, CreateViewJson, UpdateViewJSON} +import com.openbankproject.commons.model.{AccountId, AccountRoutingJsonV121, AmountOfMoneyJsonV121, BankId, CreateViewJson, ErrorMessage, UpdateViewJSON} import com.openbankproject.commons.util.ApiShortVersions import code.setup.OBPReq import org.json4s.native.Serialization.write @@ -69,6 +70,30 @@ trait V400ServerSetup extends ServerSetupWithTestData with DefaultUsers { def dynamicEndpoint_Request: OBPReq = baseRequest / "obp" / ApiShortVersions.`dynamic-endpoint`.toString def dynamicEntity_Request: OBPReq = baseRequest / "obp" / ApiShortVersions.`dynamic-entity`.toString + /** + * Shared by DynamicResourceDocTest and DynamicResourceDocJavaTest: exercises + * ResourceDoc.authCheckIO's role-gated path against a runtime-compiled dynamic-resource-doc -- + * calling without auth returns 401, authenticated without the role returns 403, and granting the + * role makes the call succeed (200), handed to `assertSuccess` for endpoint-specific checks. + */ + def assertRoleGated401Then403Then200(callUrl: OBPReq, body: String, dynamicRole: String)(assertSuccess: APIResponse => Unit): Unit = { + Then("calling without authentication returns 401") + val resp401 = makePostRequest(callUrl.POST, body) + resp401.code should equal(401) + resp401.body.extract[ErrorMessage].message should include(AuthenticatedUserIsRequired) + + Then("calling authenticated but without the role returns 403") + val resp403 = makePostRequest(callUrl.POST <@ (user1), body) + resp403.code should equal(403) + resp403.body.extract[ErrorMessage].message should include(UserHasMissingRoles) + + Then("granting the role makes the call succeed (200)") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, dynamicRole) + val resp200 = makePostRequest(callUrl.POST <@ (user1), body) + resp200.code should equal(200) + assertSuccess(resp200) + } + def randomBankId : String = { def getBanksInfo : APIResponse = { val request = v4_0_0_Request / "banks" diff --git a/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala new file mode 100644 index 0000000000..7637619c6b --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/ValidateDynamicResourceDocTest.scala @@ -0,0 +1,81 @@ +package code.api.v6_0_0 + +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.ApiRole +import code.entitlement.Entitlement +import org.json4s.native.Serialization.write + +/** + * `POST /obp/v6.0.0/management/dynamic-resource-docs/validate` must reject an unsupported + * `programming_lang` the same way `POST .../dynamic-resource-docs` (create) does, rather than + * reporting `valid = true` for a language create would actually 400 on -- see + * Http4s600.validateDynamicResourceDoc's own doc comment: CompiledObjects falls through to the + * Scala compile path for any programming_lang value it doesn't recognise as Java, so a body that + * happens to be valid Scala would otherwise "validate" successfully under a bogus/misspelled + * language. + */ +class ValidateDynamicResourceDocTest extends V600ServerSetup { + + private def validateRequest = (v6_0_0_Request / "management" / "dynamic-resource-docs" / "validate").POST <@ (user1) + + private def docWith(programmingLang: String) = + SwaggerDefinitionsJSON.jsonDynamicResourceDoc.copy( + dynamicResourceDocId = None, + programmingLang = programmingLang + ) + + feature("Validate Dynamic Resource Doc rejects an unsupported programming_lang") { + scenario("An unsupported programming_lang is rejected, not silently validated as Scala") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val resp = makePostRequest(validateRequest, write(docWith("Python"))) + + Then("the request is rejected with 400, not a 200 valid=true/false body") + resp.code should equal(400) + resp.body.toString should include("OBP-40049") + } + + scenario("programming_lang \"Scala\" is accepted (baseline, unaffected by the language check)") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val resp = makePostRequest(validateRequest, write(docWith("Scala"))) + + Then("the request reaches the compile step and responds 200") + resp.code should equal(200) + (resp.body \ "valid").values should equal(true) + } + + scenario("programming_lang \"Java\" is accepted (baseline, unaffected by the language check)") { + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + val javaBody = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class ValidateEndpointJavaProbe implements Supplier> { + | private Object apply(Object[] args) { + | Map response = new LinkedHashMap<>(); + | response.put("greeting", "hello"); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + + val doc = docWith("Java").copy(methodBody = java.net.URLEncoder.encode(javaBody, "UTF-8")) + val resp = makePostRequest(validateRequest, write(doc)) + + Then("the request reaches the compile step and responds 200") + resp.code should equal(200) + (resp.body \ "valid").values should equal(true) + } + } +} diff --git a/obp-api/src/test/scala/code/api/v7_0_0/CompileDynamicResourceDocTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/CompileDynamicResourceDocTest.scala new file mode 100644 index 0000000000..8c8e9dda03 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v7_0_0/CompileDynamicResourceDocTest.scala @@ -0,0 +1,160 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.v7_0_0 + +import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON +import code.api.util.APIUtil.OAuth._ +import code.api.util.ApiRole +import code.api.util.ErrorMessages.DynamicCodeLangNotSupport +import code.api.v7_0_0.Http4s700.Implementations7_0_0 +import code.api.v7_0_0.JSONFactory700.DynamicResourceDocCompileJsonV700 +import code.entitlement.Entitlement +import code.setup.ServerSetupWithTestData +import com.github.dwickern.macros.NameOf.nameOf +import com.openbankproject.commons.util.ApiVersion +import org.json4s.JsonAST.{JArray, JBool, JInt} +import org.json4s.native.Serialization.write +import org.scalatest.Tag + +import java.net.URLEncoder + +/** + * This suite covers the v7.0.0 dry-run compile of a Dynamic Resource Doc method body + * (POST /management/dynamic-resource-docs/compile): the role gate, and the compiler diagnostics for + * each value of programming_lang, with line numbers relative to the body the author sent. + */ +class CompileDynamicResourceDocTest extends ServerSetupWithTestData { + + object VersionOfApi extends Tag(ApiVersion.v7_0_0.toString) + object ApiEndpoint1 extends Tag(nameOf(Implementations7_0_0.compileDynamicResourceDoc)) + + def compileRequest = (baseRequest / "obp" / "v7.0.0" / "management" / "dynamic-resource-docs" / "compile").POST + + // Braced body on purpose: .github/scripts/check_test_isolation.py only recognises `def name {` as a helper. + private def dynamicCodeOn(): Unit = { + setPropsValues("allow_user_generated_scala_code" -> "true") + } + + private def grantCreateRole(): Unit = + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.canCreateDynamicResourceDoc.toString) + + private def body(methodBody: String, programmingLang: Option[String]) = write(DynamicResourceDocCompileJsonV700( + request_verb = "POST", + request_url = "/compile_test/MY_USER_ID", + method_body = URLEncoder.encode(methodBody, "UTF-8"), + // The example bodies generate the case classes the Scala example body refers to; Java ignores them. + example_request_body = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.exampleRequestBody, + success_response_body = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.successResponseBody, + programming_lang = programmingLang + )) + + private val validScalaBody = SwaggerDefinitionsJSON.jsonDynamicResourceDoc.decodedMethodBody + + // Line 9 of this body (the `return` line) refers to a variable that does not exist. + private val brokenJavaBody = + """package code.api.util.dynamic; + | + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class CompileTestBrokenJava implements Supplier> { + | @Override + | public Function get() { + | return args -> undefinedVariable; + | } + |} + |""".stripMargin + + private val validJavaBody = brokenJavaBody + .replace("CompileTestBrokenJava", "CompileTestValidJava") + .replace("undefinedVariable", "\"ok\"") + + feature("Compile Dynamic Resource Doc (dry run)") { + scenario("401 without a user, 403 without the create role", ApiEndpoint1, VersionOfApi) { + dynamicCodeOn() + makePostRequest(compileRequest, body(validScalaBody, None)).code should equal(401) + makePostRequest(compileRequest <@ (user1), body(validScalaBody, None)).code should equal(403) + } + + scenario("a Scala body compiles when programming_lang is omitted or Scala", ApiEndpoint1, VersionOfApi) { + dynamicCodeOn(); grantCreateRole() + for (lang <- List(None, Some("Scala"), Some("scala"))) { + val response = makePostRequest(compileRequest <@ (user1), body(validScalaBody, lang)) + withClue(s"programming_lang $lang, response ${response.body}: ") { + response.code should equal(200) + (response.body \ "compiles") should equal(JBool(true)) + } + } + + Given("a Scala body with a syntax error on its second line") + // A syntax error, not a type error: the Scala toolbox reports type errors without a position. + val broken = makePostRequest(compileRequest <@ (user1), body("val fine = 1\nval broken = )\nFuture.successful((fine, HttpCode.`200`(callContext.callContext)))", None)) + Then("the error is reported on line 2 of the body as sent") + withClue(s"response ${broken.body}: ") { + (broken.body \ "compiles") should equal(JBool(false)) + val errors = (broken.body \ "errors").asInstanceOf[JArray].arr + errors should not be empty + (errors.head \ "line") should equal(JInt(2)) + } + } + + scenario("a Java body is compiled by the Java compiler", ApiEndpoint1, VersionOfApi) { + dynamicCodeOn(); grantCreateRole() + Given("a valid Java body") + val valid = makePostRequest(compileRequest <@ (user1), body(validJavaBody, Some("Java"))) + Then("it compiles") + withClue(s"response ${valid.body}: ") { + valid.code should equal(200) + (valid.body \ "compiles") should equal(JBool(true)) + } + + Given("a Java body with an error on line 9") + val broken = makePostRequest(compileRequest <@ (user1), body(brokenJavaBody, Some("Java"))) + Then("the error is reported on line 9 of the body as sent, not on a line of the server's own") + withClue(s"response ${broken.body}: ") { + broken.code should equal(200) + (broken.body \ "compiles") should equal(JBool(false)) + val errors = (broken.body \ "errors").asInstanceOf[JArray].arr + errors should not be empty + (errors.head \ "line") should equal(JInt(9)) + (errors.head \ "message").values.toString should include("undefinedVariable") + } + + Given("the same valid Java body sent as Scala") + val asScala = makePostRequest(compileRequest <@ (user1), body(validJavaBody, Some("Scala"))) + Then("it does not compile: the language decides the compiler") + (asScala.body \ "compiles") should equal(JBool(false)) + } + + scenario("an unsupported programming_lang is rejected before anything is compiled", ApiEndpoint1, VersionOfApi) { + dynamicCodeOn(); grantCreateRole() + val response = makePostRequest(compileRequest <@ (user1), body(validScalaBody, Some("Cobol"))) + response.code should equal(400) + (response.body \ "message").values.toString should include(DynamicCodeLangNotSupport.takeWhile(_ != ':')) + } + } +} diff --git a/obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala index 832563ceec..09ef381885 100644 --- a/obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala +++ b/obp-api/src/test/scala/code/api/v7_0_0/DynamicChangeRequestTest.scala @@ -43,7 +43,7 @@ import org.json4s.JsonAST.{JArray, JObject} import org.json4s.native.Serialization.write import org.scalatest.Tag -import java.net.URLDecoder +import java.net.{URLDecoder, URLEncoder} /** * Maker/checker for dynamic code (docs/MAKER_CHECKER_DYNAMIC_CODE_DESIGN.md), phase 1, exercised @@ -186,7 +186,7 @@ class DynamicChangeRequestTest extends ServerSetupWithTestData { (ok.body \ "target_id").values.toString.nonEmpty should be(true) val row = storedDoc(doc.requestUrl).getOrElse(fail("doc not applied")) row.CreatedByUserId.get should be(resourceUser1.userId) - row.MethodBodyHash.get should be(APIUtil.sha256Hex(URLDecoder.decode(doc.methodBody, "UTF-8"))) + row.MethodBodyHash.get should be(APIUtil.dynamicCodeHash(doc.programmingLang, URLDecoder.decode(doc.methodBody, "UTF-8"))) row.ApprovedHash.get should be(row.MethodBodyHash.get) row.IsActive.get should be(true) MakerChecker.isExecutableDynamicResourceDoc(row.DynamicResourceDocId.get) should be(true) @@ -220,12 +220,27 @@ class DynamicChangeRequestTest extends ServerSetupWithTestData { callDynamicEndpoint("guard").codeIs(200) val row = storedDoc(doc.requestUrl).getOrElse(fail("doc not applied")) - When("the body hash is changed behind the API's back") + When("the stored hash column is changed behind the API's back") row.MethodBodyHash("tampered").save + Then("nothing changes: the guard recomputes the hash from the row, it does not trust the column") + MakerChecker.isExecutableDynamicResourceDoc(row.DynamicResourceDocId.get) should be(true) + val approvedBody = row.MethodBody.get + val approvedLang = row.Lang.get + + When("the body is changed behind the API's back") + row.MethodBody(approvedBody + URLEncoder.encode("\n// edited in the database\n", "UTF-8")).save Then("the endpoint is no longer served") MakerChecker.isExecutableDynamicResourceDoc(row.DynamicResourceDocId.get) should be(false) callDynamicEndpoint("guard").codeIs(404) - row.MethodBodyHash(row.ApprovedHash.get).save + row.MethodBody(approvedBody).save + callDynamicEndpoint("guard").codeIs(200) + + When("only the language is changed behind the API's back") + row.Lang("Java").save + Then("the approval does not carry over: it was for this body as Scala") + MakerChecker.isExecutableDynamicResourceDoc(row.DynamicResourceDocId.get) should be(false) + callDynamicEndpoint("guard").codeIs(404) + row.Lang(approvedLang).save callDynamicEndpoint("guard").codeIs(200) When("a maker without the approver role tries to deactivate") @@ -269,7 +284,7 @@ class DynamicChangeRequestTest extends ServerSetupWithTestData { When("the instance boots with approval required for the first time") MakerChecker.seedApprovedHashesIfEnabled() Then("the row's current body is treated as approved and the seed is logged") - approvedHashOf(legacy.requestUrl) should equal(APIUtil.sha256Hex(URLDecoder.decode(legacy.methodBody, "UTF-8"))) + approvedHashOf(legacy.requestUrl) should equal(APIUtil.dynamicCodeHash(legacy.programmingLang, URLDecoder.decode(legacy.methodBody, "UTF-8"))) callDynamicEndpoint("legacy").codeIs(200) logProvider.isExecuted(MakerChecker.seedMigrationName) should be(true) @@ -285,6 +300,94 @@ class DynamicChangeRequestTest extends ServerSetupWithTestData { callDynamicEndpoint("late").codeIs(404) } + scenario("a Java Dynamic Resource Doc is compiled as Java when its change request is approved", ApiEndpoint4, VersionOfApi) { + enableMakerChecker(); makerRoles(); checkerRoles() + val javaBody = + """package code.api.util.dynamic; + | + |import java.util.LinkedHashMap; + |import java.util.Map; + |import java.util.function.Function; + |import java.util.function.Supplier; + | + |public class MakerCheckerJavaDoc implements Supplier> { + | private Object apply(Object[] args) { + | @SuppressWarnings("unchecked") + | Map pathParams = (Map) args[1]; + | Map response = new LinkedHashMap<>(); + | response.put("user_id_from_path", pathParams.get("MY_USER_ID") + "_from_java"); + | return response; + | } + | + | @Override + | public Function get() { + | return this::apply; + | } + |} + |""".stripMargin + val doc = newDoc("java").copy(methodBody = URLEncoder.encode(javaBody, "UTF-8"), programmingLang = "Java") + + When("the maker submits a Java doc and a checker approves it") + val created = makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(doc)) + created.codeIs(202) + makePostRequest((v7 / "management" / "dynamic-change-requests" / str(created.body, "dynamic_change_request_id") / "approval").POST <@ (user2), + s"""{"payload_hash":"${str(created.body, "payload_hash")}"}""").codeIs(200) + + Then("the stored row is Java, its approval covers the language, and the endpoint runs the Java code") + val row = storedDoc(doc.requestUrl).getOrElse(fail("doc not applied")) + row.Lang.get should equal("Java") + row.ApprovedHash.get should equal(APIUtil.dynamicCodeHash("Java", javaBody)) + val call = callDynamicEndpoint("java") + call.codeIs(200) + str(call.body, "user_id_from_path") should equal("user-xyz_from_java") + } + + scenario("the one-off rehash carries over only approvals still valid for the row's body, and keeps pending requests fresh", VersionOfApi) { + enableMakerChecker(); makerRoles(); checkerRoles() + def approve(created: code.setup.APIResponse) = + makePostRequest((v7 / "management" / "dynamic-change-requests" / str(created.body, "dynamic_change_request_id") / "approval").POST <@ (user2), + s"""{"payload_hash":"${str(created.body, "payload_hash")}"}""").codeIs(200) + val valid = newDoc("rehashok") + approve(makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(valid))) + val edited = newDoc("rehashedited") + approve(makePostRequest((v4 / "management" / "dynamic-resource-docs").POST <@ (user1), write(edited))) + + Given("rows hashed in the old form (body only), as an instance had them before the upgrade") + def oldFormHash(row: DynamicResourceDoc) = APIUtil.sha256Hex(URLDecoder.decode(row.MethodBody.get, "UTF-8")) + val validRow = storedDoc(valid.requestUrl).get + validRow.MethodBodyHash(oldFormHash(validRow)).ApprovedHash(oldFormHash(validRow)).save + val editedRow = storedDoc(edited.requestUrl).get + editedRow.MethodBodyHash(oldFormHash(editedRow)).ApprovedHash(oldFormHash(editedRow)).save + And("one of them was edited in the database after its approval") + editedRow.MethodBody(editedRow.MethodBody.get + URLEncoder.encode("\n// edited in the database\n", "UTF-8")).save + And("an update to the valid row is pending, queued against its old-form hash") + val put = makePutRequest((v4 / "management" / "dynamic-resource-docs" / validRow.DynamicResourceDocId.get).PUT <@ (user1), + write(valid.copy(dynamicResourceDocId = Some(validRow.DynamicResourceDocId.get), summary = "pending change"))) + put.codeIs(202) + val pendingId = str(put.body, "dynamic_change_request_id") + code.dynamicchangerequest.DynamicChangeRequest.find(By(code.dynamicchangerequest.DynamicChangeRequest.DynamicChangeRequestId, pendingId)) + .foreach(_.CurrentPayloadHash(oldFormHash(storedDoc(valid.requestUrl).get)).save) + callDynamicEndpoint("rehashok").codeIs(404) + + When("the instance boots with the language-aware hash for the first time") + code.migration.MigrationScriptLog.findAll(By(code.migration.MigrationScriptLog.Name, MakerChecker.rehashMigrationName)).foreach(_.delete_!) + MakerChecker.rehashDynamicCodeWithLanguage() + + Then("the still-valid approval is moved to the new form and the endpoint is served again") + val validAfter = storedDoc(valid.requestUrl).get + val newForm = APIUtil.dynamicCodeHash(validAfter.Lang.get, URLDecoder.decode(validAfter.MethodBody.get, "UTF-8")) + validAfter.MethodBodyHash.get should equal(newForm) + validAfter.ApprovedHash.get should equal(newForm) + callDynamicEndpoint("rehashok").codeIs(200) + And("the row edited after approval stays unexecutable") + MakerChecker.isExecutableDynamicResourceDoc(storedDoc(edited.requestUrl).get.DynamicResourceDocId.get) should be(false) + And("the pending request is not stale, so it can still be approved") + approve(put) + storedDoc(valid.requestUrl).get.Summary.get should equal("pending change") + And("the rehash is logged so it never runs again") + code.migration.MigrationScriptLogProvider.migrationScriptLogProvider.vend.isExecuted(MakerChecker.rehashMigrationName) should be(true) + } + scenario("an update is queued with the live hash; rejection needs a comment and leaves the target untouched", ApiEndpoint5, VersionOfApi) { enableMakerChecker(); makerRoles(); checkerRoles() val doc = newDoc("upd") diff --git a/obp-api/src/test/scala/code/api/v7_0_0/Http4s700RoutesTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/Http4s700RoutesTest.scala index ad2f80375a..8093a019d5 100644 --- a/obp-api/src/test/scala/code/api/v7_0_0/Http4s700RoutesTest.scala +++ b/obp-api/src/test/scala/code/api/v7_0_0/Http4s700RoutesTest.scala @@ -2788,7 +2788,7 @@ class Http4s700RoutesTest extends ServerSetupWithTestData { Some(resourceUser1.userId) ).openOrThrowException("seed dynamic resource doc") val docId = seeded.dynamicResourceDocId.getOrElse(fail("seeded id")) - val expectedHash = code.api.util.APIUtil.sha256Hex(seeded.decodedMethodBody) + val expectedHash = code.api.util.APIUtil.dynamicCodeHash(seeded.programmingLang, seeded.decodedMethodBody) When("Unauthenticated GET of the list") val (unauthCode, _, _) = makeHttpRequest("/obp/v7.0.0/management/dynamic-resource-docs") @@ -2834,7 +2834,7 @@ class Http4s700RoutesTest extends ServerSetupWithTestData { Some(resourceUser1.userId) ).openOrThrowException("seed connector method") val id = seeded.connectorMethodId.getOrElse(fail("seeded id")) - val expectedHash = code.api.util.APIUtil.sha256Hex(seeded.decodedMethodBody) + val expectedHash = code.api.util.APIUtil.dynamicCodeHash(seeded.programmingLang, seeded.decodedMethodBody) addEntitlement("", resourceUser1.userId, code.api.util.ApiRole.canGetConnectorMethod.toString) val (code200, json, _) = makeHttpRequest( @@ -2853,7 +2853,7 @@ class Http4s700RoutesTest extends ServerSetupWithTestData { Some(resourceUser1.userId) ).openOrThrowException("seed dynamic message doc") val id = seeded.dynamicMessageDocId.getOrElse(fail("seeded id")) - val expectedHash = code.api.util.APIUtil.sha256Hex(seeded.decodedMethodBody) + val expectedHash = code.api.util.APIUtil.dynamicCodeHash(seeded.programmingLang, seeded.decodedMethodBody) addEntitlement("", resourceUser1.userId, code.api.util.ApiRole.canGetDynamicMessageDoc.toString) val (code200, json, _) = makeHttpRequest(