From aabb934f713ed7ac17e87f2b033e4c20675f2da7 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 4 Oct 2026 17:44:49 +0200 Subject: [PATCH 01/16] Log cache and Telemetry accept Consumer Scopes (UserOrApplication) so Platform Apps e.g. OBP Sentinel can use them --- .../scala/code/api/v5_1_0/Http4s510.scala | 6 ++++ .../code/api/v7_0_0/Http4s700Telemetry.scala | 8 +++-- .../api/v5_1_0/LogCacheEndpointTest.scala | 29 ++++++++++++++++--- .../api/v7_0_0/TelemetryEndpointTest.scala | 18 ++++++++++-- release_notes.md | 7 +++++ 5 files changed, 60 insertions(+), 8 deletions(-) diff --git a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala index 532878ac46..20fed68ddd 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala @@ -1199,6 +1199,7 @@ object Http4s510 { List($AuthenticatedUserIsRequired, UnknownError), apiTagSystem :: apiTagApi :: apiTagLogCache :: Nil, Some(List(canGetSystemLogCacheTrace, canGetSystemLogCacheAll)), + authMode = UserOrApplication, http4sPartialFunction = Some(logCacheTraceEndpoint) ) @@ -1225,6 +1226,7 @@ object Http4s510 { List($AuthenticatedUserIsRequired, UnknownError), apiTagSystem :: apiTagApi :: apiTagLogCache :: Nil, Some(List(canGetSystemLogCacheDebug, canGetSystemLogCacheAll)), + authMode = UserOrApplication, http4sPartialFunction = Some(logCacheDebugEndpoint) ) @@ -1251,6 +1253,7 @@ object Http4s510 { List($AuthenticatedUserIsRequired, UnknownError), apiTagSystem :: apiTagApi :: apiTagLogCache :: Nil, Some(List(canGetSystemLogCacheInfo, canGetSystemLogCacheAll)), + authMode = UserOrApplication, http4sPartialFunction = Some(logCacheInfoEndpoint) ) @@ -1277,6 +1280,7 @@ object Http4s510 { List($AuthenticatedUserIsRequired, UnknownError), apiTagSystem :: apiTagApi :: apiTagLogCache :: Nil, Some(List(canGetSystemLogCacheWarning, canGetSystemLogCacheAll)), + authMode = UserOrApplication, http4sPartialFunction = Some(logCacheWarningEndpoint) ) @@ -1303,6 +1307,7 @@ object Http4s510 { List($AuthenticatedUserIsRequired, UnknownError), apiTagSystem :: apiTagApi :: apiTagLogCache :: Nil, Some(List(canGetSystemLogCacheError, canGetSystemLogCacheAll)), + authMode = UserOrApplication, http4sPartialFunction = Some(logCacheErrorEndpoint) ) @@ -1329,6 +1334,7 @@ object Http4s510 { List($AuthenticatedUserIsRequired, UnknownError), apiTagSystem :: apiTagApi :: apiTagLogCache :: Nil, Some(List(canGetSystemLogCacheAll)), + authMode = UserOrApplication, http4sPartialFunction = Some(logCacheAllEndpoint) ) diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700Telemetry.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700Telemetry.scala index c297e2757f..78613576f0 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700Telemetry.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700Telemetry.scala @@ -28,7 +28,7 @@ package code.api.v7_0_0 import cats.effect.IO import code.api.Constant.ApiPathZero -import code.api.util.APIUtil.{EmptyBody, ResourceDoc} +import code.api.util.APIUtil.{EmptyBody, ResourceDoc, UserOrApplication} import code.api.util.ApiRole._ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ @@ -64,7 +64,7 @@ object Http4s700Telemetry { // Route: GET /obp/v7.0.0/management/telemetry lazy val getTelemetry: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ GET -> `prefixPath` / "management" / "telemetry" => - EndpointHelpers.withUser(req) { (_, _) => + EndpointHelpers.executeFuture(req) { val namePrefix = req.uri.query.params.get("name_prefix").filter(_.nonEmpty) Future(JSONFactory700Operations.createTelemetryJson(namePrefix)) } @@ -110,12 +110,16 @@ object Http4s700Telemetry { | |**Filter.** `name_prefix` limits the list to meters whose name starts with it, |for example `?name_prefix=obp.api.endpoint`. + | + |**Who may call it.** A User with the Role CanGetTelemetry, or an application whose Consumer holds it + |as a Scope, such as a monitoring service run as a Platform App (see ${Glossary.getGlossaryItemLink("Platform Apps")}). |""".stripMargin, EmptyBody, JSONFactory700Operations.telemetryJsonV700Example, List($AuthenticatedUserIsRequired, UserHasMissingRoles, UnknownError), List(apiTagApi, apiTagSystem), Some(List(canGetTelemetry)), + authMode = UserOrApplication, http4sPartialFunction = Some(getTelemetry) ) } diff --git a/obp-api/src/test/scala/code/api/v5_1_0/LogCacheEndpointTest.scala b/obp-api/src/test/scala/code/api/v5_1_0/LogCacheEndpointTest.scala index 4085895262..41b04c177c 100644 --- a/obp-api/src/test/scala/code/api/v5_1_0/LogCacheEndpointTest.scala +++ b/obp-api/src/test/scala/code/api/v5_1_0/LogCacheEndpointTest.scala @@ -29,10 +29,11 @@ package code.api.v5_1_0 import org.json4s._ import code.api.util.APIUtil.OAuth._ -import code.api.util.ApiRole.{CanGetSystemLogCacheAll,CanGetSystemLogCacheInfo} -import code.api.util.ErrorMessages.{UserHasMissingRoles, AuthenticatedUserIsRequired} +import code.api.util.ApiRole.{CanGetSystemLogCacheAll,CanGetSystemLogCacheError,CanGetSystemLogCacheInfo} +import code.api.util.ErrorMessages.{UserHasMissingRoles, ApplicationNotIdentified} import code.api.v5_1_0.Http4s510.Implementations5_1_0 import code.entitlement.Entitlement +import code.scope.Scope import com.github.dwickern.macros.NameOf.nameOf import com.openbankproject.commons.model.ErrorMessage import com.openbankproject.commons.util.ApiVersion @@ -57,7 +58,7 @@ class LogCacheEndpointTest extends V510ServerSetup { val response = makeGetRequest(request) Then("We should get a 401") response.code should equal(401) - response.body.extract[ErrorMessage].message should equal(AuthenticatedUserIsRequired) + response.body.extract[ErrorMessage].message should equal(ApplicationNotIdentified) } } @@ -271,4 +272,24 @@ class LogCacheEndpointTest extends V510ServerSetup { responseNegativeOffset.code should equal(400) } } -} \ No newline at end of file + + feature(s"test $ApiEndpoint1 version $VersionOfApi - Application access with a Scope") { + scenario("A Consumer holding the Role as a Scope may read the log cache without an Entitlement", ApiEndpoint1, VersionOfApi) { + val request = (v5_1_0_Request / "system" / "log-cache" / "error").GET <@(user2) + Given("user2 holds no log cache Entitlement and testConsumer2 no Scope") + makeGetRequest(request).code should equal(403) + + When("testConsumer2, which user2 signs with, is granted CanGetSystemLogCacheError as a Scope") + val granted = Scope.scope.vend.addScope("", testConsumer2.id.get.toString, CanGetSystemLogCacheError.toString) + try { + val response = makeGetRequest(request) + Then("the error level log cache is returned") + response.code should equal(200) + (response.body \ "entries") shouldBe a[JArray] + + And("the Scope covers only its own level") + makeGetRequest((v5_1_0_Request / "system" / "log-cache" / "info").GET <@(user2)).code should equal(403) + } finally Scope.scope.vend.deleteScope(granted) + } + } +} diff --git a/obp-api/src/test/scala/code/api/v7_0_0/TelemetryEndpointTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/TelemetryEndpointTest.scala index c59f9cbe88..5fb2cc99b6 100644 --- a/obp-api/src/test/scala/code/api/v7_0_0/TelemetryEndpointTest.scala +++ b/obp-api/src/test/scala/code/api/v7_0_0/TelemetryEndpointTest.scala @@ -30,9 +30,10 @@ package code.api.v7_0_0 import code.api.Constant import code.api.util.APIUtil.OAuth._ import code.api.util.ApiRole.CanGetTelemetry -import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, UserHasMissingRoles} +import code.api.util.ErrorMessages.{ApplicationNotIdentified, UserHasMissingRoles} import code.api.v6_0_0.V600ServerSetup import code.entitlement.Entitlement +import code.scope.Scope import com.openbankproject.commons.model.ErrorMessage import com.openbankproject.commons.util.ApiVersion import org.scalatest.Tag @@ -62,7 +63,7 @@ class TelemetryEndpointTest extends V600ServerSetup { scenario("Anonymous access fails with 401", ApiEndpoint, VersionOfApi) { val response = makeGetRequest(telemetryRequest.GET) response.code should equal(401) - response.body.extract[ErrorMessage].message should equal(AuthenticatedUserIsRequired) + response.body.extract[ErrorMessage].message should equal(ApplicationNotIdentified) } scenario("A logged-in user without CanGetTelemetry gets 403", ApiEndpoint, VersionOfApi) { @@ -135,5 +136,18 @@ class TelemetryEndpointTest extends V600ServerSetup { names should not be empty all(names) should startWith("jvm.memory") } + + scenario("A Consumer holding CanGetTelemetry as a Scope may read Telemetry without an Entitlement", ApiEndpoint, VersionOfApi) { + Given("user2 holds no CanGetTelemetry Entitlement and testConsumer2 no Scope") + makeGetRequest(telemetryRequest.GET <@ (user2)).code should equal(403) + + When("testConsumer2, which user2 signs with, is granted CanGetTelemetry as a Scope") + val granted = Scope.scope.vend.addScope("", testConsumer2.id.get.toString, CanGetTelemetry.toString) + val response = try makeGetRequest(telemetryRequest.GET <@ (user2)) finally Scope.scope.vend.deleteScope(granted) + + Then("Telemetry is returned") + response.code should equal(200) + response.body.extract[TelemetryJsonV700].api_instance_id should equal(Constant.ApiInstanceId) + } } } diff --git a/release_notes.md b/release_notes.md index c1a28aedef..136e0e2114 100644 --- a/release_notes.md +++ b/release_notes.md @@ -3,6 +3,13 @@ ### Most recent changes at top of file ``` Date Commit Action +04/10/2026 TBD CHANGED: GET /obp/v5.1.0/system/log-cache/LEVEL (trace, debug, info, warning, + error, all) and GET /obp/v7.0.0/management/telemetry accept a Consumer that + holds the Role as a Scope, as well as a User who holds it as an Entitlement + (authMode UserOrApplication), so a monitoring service can read them as a + Platform App with its own application token. A call with no credentials + still gets 401, now with ApplicationNotIdentified instead of + AuthenticatedUserIsRequired. Response bodies are unchanged. 03/10/2026 TBD CHANGED: a Dynamic Resource Doc is served under its space, at /obp/dynamic-endpoint/banks/BANK_ID/dynamic-resource-doc/REQUEST_URL, with SYS for the system space. A verb and URL are unique within a space instead of From c7c315d78c6901cfa67b15a92629d0a99c276720 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 4 Oct 2026 17:50:10 +0200 Subject: [PATCH 02/16] API Metrics record domain_api_url: the path and query string a caller used under a Domain API, before it was rewritten to the OBP URL kept in url; empty for other calls. Stored in metric and metricarchive, returned by v6.0.0 GET /management/metrics, carried on the metrics gRPC stream (MetricEvent field 23), and filterable with domain_api_url (starts with). Fix: the metric list now applies the documented consent_reference_id and certificate_trust filters, which were silently ignored. Tests: DomainApisTest (metric rows of Domain API calls), MetricEventFieldsTest (field 23), MetricFilterParamsTest. --- .../src/main/protobuf/metrics_stream.proto | 3 ++ .../SwaggerDefinitionsJSON.scala | 3 +- .../dynamic/domainapi/DomainApiPaths.scala | 7 ++- .../dynamic/domainapi/Http4sDomainApi.scala | 2 +- .../main/scala/code/api/util/APIUtil.scala | 12 ++++- .../main/scala/code/api/util/ApiSession.scala | 10 +++- .../main/scala/code/api/util/Glossary.scala | 1 + .../main/scala/code/api/util/OBPParam.scala | 2 + .../scala/code/api/util/WriteMetricUtil.scala | 12 +++-- .../code/api/util/http4s/Http4sSupport.scala | 1 + .../scala/code/api/v6_0_0/Http4s600.scala | 2 + .../code/api/v6_0_0/JSONFactory6.0.0.scala | 8 ++- .../main/scala/code/metrics/APIMetrics.scala | 10 +++- .../code/metrics/ElasticsearchMetrics.scala | 5 +- .../scala/code/metrics/MappedMetrics.scala | 24 +++++++-- .../code/metrics/MetricBatchWriter.scala | 40 ++++++++------ .../MetricsStreamServiceImpl.scala | 3 +- .../grpc/metricsstream/api/MetricEvent.scala | 21 ++++++-- .../api/MetricsStreamProto.scala | 1 + .../scheduler/MetricsArchiveScheduler.scala | 3 +- .../api/util/MetricFilterParamsTest.scala | 53 +++++++++++++++++++ .../code/api/v7_0_0/DomainApisTest.scala | 12 +++++ .../metricsstream/MetricEventFieldsTest.scala | 13 +++-- 23 files changed, 202 insertions(+), 46 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/util/MetricFilterParamsTest.scala diff --git a/obp-api/src/main/protobuf/metrics_stream.proto b/obp-api/src/main/protobuf/metrics_stream.proto index 4b9354472c..ba681f87e8 100644 --- a/obp-api/src/main/protobuf/metrics_stream.proto +++ b/obp-api/src/main/protobuf/metrics_stream.proto @@ -60,6 +60,9 @@ message MetricEvent { // The specifics behind certificate_trust: the forwarding proxy's subject for // "forwarded", the rejection reason for "none". Matches MetricJsonV600.certificate_trust_detail. string certificate_trust_detail = 22; + // For a call made under a Domain API, the path and query string the caller used, before it was + // rewritten to the OBP URL in `url`; empty for every other call. Matches MetricJsonV600.domain_api_url. + string domain_api_url = 23; } // Live tail of API metrics as they are written. diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala index 65a1237637..a47bb6fabe 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala @@ -3232,7 +3232,8 @@ object SwaggerDefinitionsJSON { consent_reference_id = Some(ExampleValue.consentReferenceIdExample.value), auth_type = Some("Consent"), certificate_trust = Some("forwarded"), - certificate_trust_detail = Some("cn=nginx-prod-1,ou=edge,o=tesobe gmbh,c=de") + certificate_trust_detail = Some("cn=nginx-prod-1,ou=edge,o=tesobe gmbh,c=de"), + domain_api_url = None ) lazy val metricsJsonV600 = MetricsJsonV600( metrics = List(metricJsonV600) diff --git a/obp-api/src/main/scala/code/api/dynamic/domainapi/DomainApiPaths.scala b/obp-api/src/main/scala/code/api/dynamic/domainapi/DomainApiPaths.scala index faa606a3a7..32ada194e9 100644 --- a/obp-api/src/main/scala/code/api/dynamic/domainapi/DomainApiPaths.scala +++ b/obp-api/src/main/scala/code/api/dynamic/domainapi/DomainApiPaths.scala @@ -54,8 +54,11 @@ import org.json4s.JsonAST.{JObject, JValue} */ object DomainApiPaths { - /** What the front door records on a request it rewrote: which Domain API, and the path that was called. */ - case class DomainApiCall(domainApiId: String, basePath: String, calledPath: String) + /** + * What the front door records on a request it rewrote: which Domain API, and the URL that was called (path + * and query string, the shape of CallContext.url), which API Metrics record as `domain_api_url`. + */ + case class DomainApiCall(domainApiId: String, basePath: String, calledUrl: String) val domainApiCallKey: org.typelevel.vault.Key[DomainApiCall] = org.typelevel.vault.Key.newKey[IO, DomainApiCall].unsafeRunSync() diff --git a/obp-api/src/main/scala/code/api/dynamic/domainapi/Http4sDomainApi.scala b/obp-api/src/main/scala/code/api/dynamic/domainapi/Http4sDomainApi.scala index c71d1919f4..54a3f8210b 100644 --- a/obp-api/src/main/scala/code/api/dynamic/domainapi/Http4sDomainApi.scala +++ b/obp-api/src/main/scala/code/api/dynamic/domainapi/Http4sDomainApi.scala @@ -93,7 +93,7 @@ object Http4sDomainApi extends MdcLoggable { case Nil => OptionT.none[IO, Response[IO]] case _ => val marked = req.withAttribute(domainApiCallKey, - DomainApiCall(route.domainApiId, route.basePath, req.uri.path.renderString)) + DomainApiCall(route.domainApiId, route.basePath, req.uri.renderString)) Http4sDynamicEndpoint.wrappedRoutesDynamicEndpoint.run(withPath(marked, DomainApiPaths.dynamicResourceDocPath(route.bankId, rest))) .orElse(Http4sDynamicEntity.wrappedRoutesDynamicEntityV700.run(withPath(marked, DomainApiPaths.dynamicEntityPath(route.bankId, rest)))) } diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index a8c6ab2ead..962cfb6c7a 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -1233,6 +1233,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ case "consent_id" => Full(OBPConsentId(values.head)) case "consent_reference_id" => Full(OBPConsentReferenceId(values.head)) case "certificate_trust" => Full(OBPCertificateTrust(values.head)) + case "domain_api_url" => Full(OBPDomainApiUrl(values.head)) case "user_id" => Full(OBPUserId(values.head)) case "provider_provider_id" => Full(ProviderProviderId(values.head)) case "bank_id" => Full(OBPBankId(values.head)) @@ -1315,6 +1316,11 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ username <- getHttpParamValuesByName(httpParams, "username") email <- getHttpParamValuesByName(httpParams, "email") httpStatusCode <- getHttpParamValuesByName(httpParams, "http_status_code") + // Read here so that the metrics endpoints actually apply these filters: getHttpParamValuesByName maps + // each name to its query param, but a name not read in this list never reaches a query. + consentReferenceId <- getHttpParamValuesByName(httpParams, "consent_reference_id") + certificateTrust <- getHttpParamValuesByName(httpParams, "certificate_trust") + domainApiUrl <- getHttpParamValuesByName(httpParams, "domain_api_url") }yield{ // Extract the sort field name from the sort_by query param (e.g. "url", "date"). // OBPOrdering expects Option[String], but sortBy is an OBPQueryParam. @@ -1328,7 +1334,8 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ anon, status, consumerId, azp, iss, consentId, userId, providerProviderId, url, appName, implementedByPartialFunction, implementedInVersion, verb, correlationId, duration, httpStatusCode, excludeAppNames, excludeUrlPattern, excludeImplementedByPartialfunctions, includeAppNames, includeUrlPattern, includeImplementedByPartialfunctions, - connectorName,functionName, bankId, accountId, customerId, lockedStatus, roleName, provider, username, email, deletedStatus + connectorName,functionName, bankId, accountId, customerId, lockedStatus, roleName, provider, username, email, deletedStatus, + consentReferenceId, certificateTrust, domainApiUrl ).filter(_ != OBPEmpty()) } } @@ -1369,6 +1376,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ val consentId = getHttpRequestUrlParam(httpRequestUrl,"consent_id") val consentReferenceId = getHttpRequestUrlParam(httpRequestUrl,"consent_reference_id") val certificateTrust = getHttpRequestUrlParam(httpRequestUrl,"certificate_trust") + val domainApiUrl = getHttpRequestUrlParam(httpRequestUrl,"domain_api_url") val userId = getHttpRequestUrlParam(httpRequestUrl, "user_id") val providerProviderId = getHttpRequestUrlParam(httpRequestUrl, "provider_provider_id") val bankId = getHttpRequestUrlParam(httpRequestUrl, "bank_id") @@ -1404,7 +1412,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ Full(List( HTTPParam("sort_by",sortBy), HTTPParam("sort_direction",sortDirection), HTTPParam("from_date",fromDate), HTTPParam("to_date", toDate), HTTPParam("limit",limit), HTTPParam("offset",offset), - HTTPParam("anon", anon), HTTPParam("status", status), HTTPParam("consumer_id", consumerId), HTTPParam("azp", azp), HTTPParam("iss", iss), HTTPParam("consent_id", consentId), HTTPParam("consent_reference_id", consentReferenceId), HTTPParam("certificate_trust", certificateTrust), HTTPParam("user_id", userId), HTTPParam("provider_provider_id", providerProviderId), HTTPParam("url", url), HTTPParam("app_name", appName), + HTTPParam("anon", anon), HTTPParam("status", status), HTTPParam("consumer_id", consumerId), HTTPParam("azp", azp), HTTPParam("iss", iss), HTTPParam("consent_id", consentId), HTTPParam("consent_reference_id", consentReferenceId), HTTPParam("certificate_trust", certificateTrust), HTTPParam("domain_api_url", domainApiUrl), HTTPParam("user_id", userId), HTTPParam("provider_provider_id", providerProviderId), HTTPParam("url", url), HTTPParam("app_name", appName), HTTPParam("implemented_by_partial_function",implementedByPartialFunction), HTTPParam("implemented_in_version",implementedInVersion), HTTPParam("verb", verb), HTTPParam("correlation_id", correlationId), HTTPParam("duration", duration), HTTPParam("exclude_app_names", excludeAppNames), HTTPParam("exclude_url_patterns", excludeUrlPattern),HTTPParam("exclude_implemented_by_partial_functions", excludeImplementedByPartialfunctions), diff --git a/obp-api/src/main/scala/code/api/util/ApiSession.scala b/obp-api/src/main/scala/code/api/util/ApiSession.scala index 656801e885..8f5894b297 100644 --- a/obp-api/src/main/scala/code/api/util/ApiSession.scala +++ b/obp-api/src/main/scala/code/api/util/ApiSession.scala @@ -87,6 +87,9 @@ case class CallContext( // The hops the request passed through: the X-Forwarded-For chain it arrived // with, followed by the TCP peer. Recorded in API Metrics; see RemoteIpUtil. forwardedFor: String = "", + // The path and query string a caller used under a Domain API, before the request was + // rewritten to the OBP URL in `url`; None for every other call. Recorded in API Metrics. + domainApiUrl: Option[String] = None, resourceDocument: Option[ResourceDoc] = None, startTime: Option[Date] = Some(Helpers.now), endTime: Option[Date] = None, @@ -250,7 +253,8 @@ case class CallContext( certificateTrust = this.certificateTrust, certificateTrustDetail = this.certificateTrustDetail, ipAddress = this.ipAddress, - forwardedFor = this.forwardedFor + forwardedFor = this.forwardedFor, + domainApiUrl = this.domainApiUrl ) } @@ -353,7 +357,9 @@ case class CallContextLight(gatewayLoginRequestPayload: Option[PayloadOfJwtJSON] // The client address OBP-API decided on (CallContext.ipAddress) ipAddress: String = "", // The hops the request passed through (CallContext.forwardedFor) - forwardedFor: String = "" + forwardedFor: String = "", + // The URL called under a Domain API (CallContext.domainApiUrl) + domainApiUrl: Option[String] = None ) trait LoginParam diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 27ed25da3e..2c6d459ef2 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -6957,6 +6957,7 @@ object Glossary extends MdcLoggable { |- `source_ip`: the address of the client, as OBP-API decided it (see [Client IP Address](/glossary#Client-IP-Address)). Records written before this was introduced hold the raw `X-Forwarded-For` header instead |- `forwarded_for`: the hops the call passed through: the `X-Forwarded-For` list it arrived with, followed by the address of the machine that connected to OBP-API. Entries to the left of the first address OBP-API does not trust may have been written by the caller, so read them as a claim, not a fact |- `target_ip`: the `X-Forwarded-Host` request header, as sent + |- `domain_api_url`: for a call made under a [Domain API](/glossary#Domain-APIs), the path and query string the caller used, such as `/carbon-registry/v1/activity?limit=10`; the URL above is then the OBP URL the call was served at. Absent for every other call. A Domain API's base paths never overlap, so the calls through one Domain API are those whose `domain_api_url` starts with its base path. |- the `api_instance_id` of the OBP-API instance that served the call |- the response body, for selected endpoints only | diff --git a/obp-api/src/main/scala/code/api/util/OBPParam.scala b/obp-api/src/main/scala/code/api/util/OBPParam.scala index 12f4a46b24..ef289927d6 100644 --- a/obp-api/src/main/scala/code/api/util/OBPParam.scala +++ b/obp-api/src/main/scala/code/api/util/OBPParam.scala @@ -96,6 +96,8 @@ case class OBPConsentId(value: String) extends OBPQueryParam case class OBPConsentReferenceId(value: String) extends OBPQueryParam // PeerTrust.Resolution.mode on the metric row: "direct", "forwarded" or "none". case class OBPCertificateTrust(value: String) extends OBPQueryParam +// Metrics whose domain_api_url starts with this value: the calls made under a Domain API, or under one of its paths. +case class OBPDomainApiUrl(value: String) extends OBPQueryParam case class OBPUserId(value: String) extends OBPQueryParam // Multiple user ids, matched with SQL IN — used by self-service endpoints that lock the // user filter to a server-resolved set (e.g. /my/metrics: the human plus their consent-agents). diff --git a/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala b/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala index 8b6ccbf461..9fa52e49e2 100644 --- a/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala +++ b/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala @@ -103,7 +103,8 @@ object WriteMetricUtil extends MdcLoggable { publishMetricEvent(userId, cc.url, cc.startTime.getOrElse(null), duration, userName, appName, developerEmail, consumerId, implementedByPartialFunction, cc.implementedInVersion, cc.verb, cc.httpCode, cc.correlationId, sourceIp, targetIp, forwardedFor, cc.operationId.getOrElse(""), - cc.consentReferenceId.orNull, cc.certificateTrust.orNull, cc.certificateTrustDetail.orNull, authType) + cc.consentReferenceId.orNull, cc.certificateTrust.orNull, cc.certificateTrustDetail.orNull, authType, + cc.domainApiUrl.orNull) } } @@ -175,7 +176,8 @@ object WriteMetricUtil extends MdcLoggable { cc.consentReferenceId.orNull, cc.certificateTrust.orNull, cc.certificateTrustDetail.orNull, - authType + authType, + cc.domainApiUrl.orNull ) } catch { case NonFatal(e) => @@ -211,7 +213,8 @@ object WriteMetricUtil extends MdcLoggable { consentReferenceId: String, certificateTrust: String, certificateTrustDetail: String, - authType: String): Unit = { + authType: String, + domainApiUrl: String): Unit = { if (!MetricsEventBus.isEnabled) return try { implicit val fmts = metricFormats @@ -240,7 +243,8 @@ object WriteMetricUtil extends MdcLoggable { "consent_reference_id" -> Option(consentReferenceId).getOrElse(""), "certificate_trust" -> Option(certificateTrust).getOrElse(""), "certificate_trust_detail" -> Option(certificateTrustDetail).getOrElse(""), - "auth_type" -> Option(authType).getOrElse("") + "auth_type" -> Option(authType).getOrElse(""), + "domain_api_url" -> Option(domainApiUrl).getOrElse("") )) MetricsEventBus.publish(payload) } catch { diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala index 09c138b3af..a94c481123 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala @@ -697,6 +697,7 @@ object Http4sCallContextBuilder { } } yield CallContext( url = request.uri.renderString, + domainApiUrl = request.attributes.lookup(code.api.dynamic.domainapi.DomainApiPaths.domainApiCallKey).map(_.calledUrl), forwardedFor = RemoteIpUtil.forwardedForPath( request.remoteAddr.map(_.toUriString).getOrElse(""), requestHeaderValues(request, "X-Forwarded-For") diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index c90e6eedf6..95908b6ea4 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -7830,6 +7830,8 @@ object Http4s600 { | |18 certificate_trust (if null ignore) - Returns calls by how the caller's certificate was established: direct (the TLS peer was the caller), forwarded (a trusted proxy forwarded the caller's certificate) or none (certificate material was present but no caller was identified). eg: certificate_trust=forwarded | + |19 domain_api_url (if null ignore) - Returns calls made under a Domain API whose called URL (the metric's domain_api_url) starts with this value: a base path gives every call through that Domain API, a longer value the calls to one of its paths. eg: domain_api_url=/carbon-registry/v1/ + | """.stripMargin, EmptyBody, metricsJsonV600, diff --git a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala index 750c3c9c43..cd491e44a3 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala @@ -486,7 +486,10 @@ case class MetricJsonV600( // absent when the request carried no certificate material. See PeerTrust.Resolution. certificate_trust: Option[String], // The forwarding proxy's subject DN, or the reason no caller was identified. - certificate_trust_detail: Option[String] + certificate_trust_detail: Option[String], + // The path and query string the caller used under a Domain API, before it was rewritten to the OBP + // URL in `url`. Absent for every call that did not come through a Domain API. + domain_api_url: Option[String] = None ) case class MetricsJsonV600(metrics: List[MetricJsonV600]) @@ -1779,7 +1782,8 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { consent_reference_id = Option(metric.getConsentReferenceId()).filter(_.nonEmpty), auth_type = Option(metric.getAuthType()).filter(_.nonEmpty), certificate_trust = Option(metric.getCertificateTrust()).filter(_.nonEmpty), - certificate_trust_detail = Option(metric.getCertificateTrustDetail()).filter(_.nonEmpty) + certificate_trust_detail = Option(metric.getCertificateTrustDetail()).filter(_.nonEmpty), + domain_api_url = Option(metric.getDomainApiUrl()).filter(_.nonEmpty) ) } diff --git a/obp-api/src/main/scala/code/metrics/APIMetrics.scala b/obp-api/src/main/scala/code/metrics/APIMetrics.scala index 1177efa07b..2b53007666 100644 --- a/obp-api/src/main/scala/code/metrics/APIMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/APIMetrics.scala @@ -130,7 +130,9 @@ trait APIMetrics { consentReferenceId: String, certificateTrust: String, certificateTrustDetail: String, - authType: String): Unit + authType: String, + // The URL called under a Domain API (CallContext.domainApiUrl); null for every other call. + domainApiUrl: String = null): Unit def saveMetricsArchive(primaryKey: Long, userId: String, @@ -154,7 +156,8 @@ trait APIMetrics { consentReferenceId: String, certificateTrust: String, certificateTrustDetail: String, - authType: String + authType: String, + domainApiUrl: String = null ): Boolean // //TODO: ordering of list? should this be by date? currently not enforced @@ -211,6 +214,9 @@ trait APIMetric { def getSourceIp(): String def getTargetIp(): String def getForwardedFor(): String + // The path and query string called under a Domain API, before it was rewritten to the OBP URL in + // getUrl. Null for every call that did not come through a Domain API. + def getDomainApiUrl(): String def getApiInstanceId(): String def getConsentReferenceId(): String def getCertificateTrust(): String diff --git a/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala b/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala index a16466fd90..02cbd0a1f7 100644 --- a/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala @@ -43,7 +43,7 @@ object ElasticsearchMetrics extends APIMetrics { override def saveMetric(userId: String, url: String, date: Date, duration: Long, userName: String, appName: String, developerEmail: String, consumerId: String, implementedByPartialFunction: String, implementedInVersion: String, verb: String, httpCode: Option[Int], correlationId: String, responseBody: String, sourceIp: String, targetIp: String, forwardedFor: String, apiInstanceId: String, consentReferenceId: String, certificateTrust: String, certificateTrustDetail: String, - authType: String): Unit = { + authType: String, domainApiUrl: String): Unit = { if (APIUtil.getPropsAsBoolValue("allow_elasticsearch", false) && APIUtil.getPropsAsBoolValue("allow_elasticsearch_metrics", false) ) { //TODO ,need to be fixed now add more parameters es.indexMetric(userId, url, date, duration, userName, appName, developerEmail, correlationId, apiInstanceId) @@ -58,7 +58,8 @@ object ElasticsearchMetrics extends APIMetrics { consentReferenceId: String, certificateTrust: String, certificateTrustDetail: String, - authType: String): Boolean = ??? + authType: String, + domainApiUrl: String): Boolean = ??? // override def getAllGroupedByUserId(): Map[String, List[APIMetric]] = { // //TODO: replace the following with valid ES query diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala index b156e2ed56..4d73b046bf 100644 --- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala @@ -142,7 +142,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ override def saveMetric(userId: String, url: String, date: Date, duration: Long, userName: String, appName: String, developerEmail: String, consumerId: String, implementedByPartialFunction: String, implementedInVersion: String, verb: String, httpCode: Option[Int], correlationId: String, responseBody: String, sourceIp: String, targetIp: String, forwardedFor: String, apiInstanceId: String, consentReferenceId: String, certificateTrust: String, certificateTrustDetail: String, - authType: String): Unit = { + authType: String, domainApiUrl: String): Unit = { // A correlation id is expected on every metric. Rows without one cannot be moved // to the archive later (its correlationId column requires a UUID), so flag it at // write time where the source of the missing id can actually be traced. @@ -172,7 +172,8 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ consentReferenceId = consentReferenceId, certificateTrust = certificateTrust, certificateTrustDetail = certificateTrustDetail, - authType = authType + authType = authType, + domainApiUrl = domainApiUrl ) ) } @@ -184,7 +185,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ responseBody: String, sourceIp: String, targetIp: String, forwardedFor: String, apiInstanceId: String, consentReferenceId: String, certificateTrust: String, certificateTrustDetail: String, - authType: String): Boolean = { + authType: String, domainApiUrl: String): Boolean = { // Fix: dedup by the source metric's primary key stored in `metricId`, NOT by the // archive's own auto-increment `id`. The two are unrelated id-spaces; matching on // `id` overwrites an unrelated archived row once the archive's id sequence grows @@ -214,6 +215,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ .certificateTrust(certificateTrust) .certificateTrustDetail(certificateTrustDetail) .authType(authType) + .domainApiUrl(domainApiUrl) httpCode match { case Some(code) => metric.httpCode(code) @@ -323,6 +325,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val httpStatusCode = queryParams.collect { case OBPHttpStatusCode(value) => By(MappedMetric.httpCode, value) }.headOption val consentReferenceId = queryParams.collect { case OBPConsentReferenceId(value) => By(MappedMetric.consentReferenceId, value) }.headOption val certificateTrust = queryParams.collect { case OBPCertificateTrust(value) => By(MappedMetric.certificateTrust, value) }.headOption + val domainApiUrl = queryParams.collect { case OBPDomainApiUrl(value) => Like(MappedMetric.domainApiUrl, s"$value%") }.headOption val anon = queryParams.collect { case OBPAnon(true) => By(MappedMetric.userId, "null") case OBPAnon(false) => NotBy(MappedMetric.userId, "null") @@ -352,6 +355,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ httpStatusCode.toSeq, consentReferenceId.toSeq, certificateTrust.toSeq, + domainApiUrl.toSeq, anon.toSeq, excludeAppNames.toSeq.flatten ).flatten @@ -854,6 +858,12 @@ class MappedMetric extends APIMetric with LongKeyedMapper[MappedMetric] with IdP override def dbColumnName = "forwarded_for" override def defaultValue = null } + // The path and query string a caller used under a Domain API, before the request was rewritten to + // the OBP URL in `url`. Null for every other call. The same width as `url`, in both tables. + object domainApiUrl extends MappedString(this, 2000) { + override def dbColumnName = "domain_api_url" + override def defaultValue = null + } object apiInstanceId extends MappedString(this, 255) // Set when the request was authenticated via a consent. Null otherwise. object consentReferenceId extends MappedString(this, 36) { @@ -899,6 +909,7 @@ class MappedMetric extends APIMetric with LongKeyedMapper[MappedMetric] with IdP override def getSourceIp(): String = sourceIp.get override def getTargetIp(): String = targetIp.get override def getForwardedFor(): String = forwardedFor.get + override def getDomainApiUrl(): String = domainApiUrl.get override def getApiInstanceId(): String = apiInstanceId.get override def getConsentReferenceId(): String = consentReferenceId.get override def getCertificateTrust(): String = certificateTrust.get @@ -969,6 +980,12 @@ class MetricArchive extends APIMetric with LongKeyedMapper[MetricArchive] with I override def dbColumnName = "forwarded_for" override def defaultValue = null } + // The path and query string a caller used under a Domain API, before the request was rewritten to + // the OBP URL in `url`. Null for every other call. The same width as `url`, in both tables. + object domainApiUrl extends MappedString(this, 2000) { + override def dbColumnName = "domain_api_url" + override def defaultValue = null + } object apiInstanceId extends MappedString(this, 255) // Set when the request was authenticated via a consent. Null otherwise. object consentReferenceId extends MappedString(this, 36) { @@ -1012,6 +1029,7 @@ class MetricArchive extends APIMetric with LongKeyedMapper[MetricArchive] with I override def getSourceIp(): String = sourceIp.get override def getTargetIp(): String = targetIp.get override def getForwardedFor(): String = forwardedFor.get + override def getDomainApiUrl(): String = domainApiUrl.get override def getApiInstanceId(): String = apiInstanceId.get override def getConsentReferenceId(): String = consentReferenceId.get override def getCertificateTrust(): String = certificateTrust.get diff --git a/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala b/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala index 8396f0709b..012498078c 100644 --- a/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala +++ b/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala @@ -72,7 +72,24 @@ object MetricBatchWriter extends MdcLoggable { consentReferenceId: String, certificateTrust: String, certificateTrustDetail: String, - authType: String + authType: String, + // The URL called under a Domain API; null for every other call. + domainApiUrl: String = null + ) + + /** + * This holds one row's values in the order of the INSERT's columns. It is a case class, not a tuple, + * because the row has more than 22 values and Scala tuples stop at 22. Each text value is an Option so + * that Doobie writes a null as SQL NULL (its Put[String] refuses a null). + */ + private case class InsertValues( + userId: Option[String], url: Option[String], date: Timestamp, duration: Long, userName: Option[String], + appName: Option[String], developerEmail: Option[String], consumerId: Option[String], + implementedByPartialFunction: Option[String], implementedInVersion: Option[String], verb: Option[String], + httpCode: Int, correlationId: Option[String], responseBody: Option[String], sourceIp: Option[String], + targetIp: Option[String], forwardedFor: Option[String], apiInstanceId: Option[String], + consentReferenceId: Option[String], certificateTrust: Option[String], certificateTrustDetail: Option[String], + authType: Option[String], domainApiUrl: Option[String] ) private val queue = new ConcurrentLinkedQueue[MetricRow]() @@ -145,7 +162,8 @@ object MetricBatchWriter extends MdcLoggable { consentReferenceId = fit(row.consentReferenceId, table.consentReferenceId), certificateTrust = fit(row.certificateTrust, table.certificateTrust), certificateTrustDetail = fit(row.certificateTrustDetail, table.certificateTrustDetail), - authType = fit(row.authType, table.authType) + authType = fit(row.authType, table.authType), + domainApiUrl = fit(row.domainApiUrl, table.domainApiUrl) ) } @@ -179,22 +197,14 @@ object MetricBatchWriter extends MdcLoggable { developeremail, consumerid, implementedbypartialfunction, implementedinversion, verb, httpcode, correlationid, responsebody, sourceip, targetip, forwarded_for, apiinstanceid, consent_reference_id, - certificate_trust, certificate_trust_detail, auth_type - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + certificate_trust, certificate_trust_detail, auth_type, domain_api_url + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """ - // Use Option[String] so Doobie handles nullable fields via Put[Option[String]] - // instead of Put[String] which throws "oops, null" on null values - val insert = Update[ - (Option[String], Option[String], Timestamp, Long, Option[String], Option[String], - Option[String], Option[String], Option[String], - Option[String], Option[String], Int, Option[String], - Option[String], Option[String], Option[String], Option[String], Option[String], Option[String], - Option[String], Option[String], Option[String]) - ](insertSql) + val insert = Update[InsertValues](insertSql) val values = rows.map { r => - ( + InsertValues( Option(r.userId), Option(r.url), new Timestamp(if (r.date != null) r.date.getTime else 0L), r.duration, Option(r.userName), Option(r.appName), Option(r.developerEmail), Option(r.consumerId), Option(r.implementedByPartialFunction), @@ -202,7 +212,7 @@ object MetricBatchWriter extends MdcLoggable { Option(r.responseBody), Option(r.sourceIp), Option(r.targetIp), Option(r.forwardedFor), Option(r.apiInstanceId), Option(r.consentReferenceId), Option(r.certificateTrust), Option(r.certificateTrustDetail), - Option(r.authType) + Option(r.authType), Option(r.domainApiUrl) ) } diff --git a/obp-api/src/main/scala/code/obp/grpc/metricsstream/MetricsStreamServiceImpl.scala b/obp-api/src/main/scala/code/obp/grpc/metricsstream/MetricsStreamServiceImpl.scala index 3bab0d6186..4386674b8a 100644 --- a/obp-api/src/main/scala/code/obp/grpc/metricsstream/MetricsStreamServiceImpl.scala +++ b/obp-api/src/main/scala/code/obp/grpc/metricsstream/MetricsStreamServiceImpl.scala @@ -146,7 +146,8 @@ object MetricsStreamServiceImpl extends MetricsStreamServiceGrpc.MetricsStreamSe forwardedFor = (jv \ "forwarded_for").extractOrElse[String](""), authType = (jv \ "auth_type").extractOrElse[String](""), certificateTrust = (jv \ "certificate_trust").extractOrElse[String](""), - certificateTrustDetail = (jv \ "certificate_trust_detail").extractOrElse[String]("") + certificateTrustDetail = (jv \ "certificate_trust_detail").extractOrElse[String](""), + domainApiUrl = (jv \ "domain_api_url").extractOrElse[String]("") ) } } diff --git a/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricEvent.scala b/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricEvent.scala index d51804cb63..38b15627e7 100644 --- a/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricEvent.scala +++ b/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricEvent.scala @@ -55,7 +55,8 @@ final case class MetricEvent( forwardedFor: _root_.scala.Predef.String = "", authType: _root_.scala.Predef.String = "", certificateTrust: _root_.scala.Predef.String = "", - certificateTrustDetail: _root_.scala.Predef.String = "" + certificateTrustDetail: _root_.scala.Predef.String = "", + domainApiUrl: _root_.scala.Predef.String = "" ) extends scalapb.GeneratedMessage with scalapb.Message[MetricEvent] with scalapb.lenses.Updatable[MetricEvent] { @transient private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0 @@ -83,6 +84,7 @@ final case class MetricEvent( if (authType != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(20, authType) } if (certificateTrust != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(21, certificateTrust) } if (certificateTrustDetail != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(22, certificateTrustDetail) } + if (domainApiUrl != "") { __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(23, domainApiUrl) } __size } final override def serializedSize: _root_.scala.Int = { @@ -116,6 +118,7 @@ final case class MetricEvent( { val __v = authType; if (__v != "") _output__.writeString(20, __v) }; { val __v = certificateTrust; if (__v != "") _output__.writeString(21, __v) }; { val __v = certificateTrustDetail; if (__v != "") _output__.writeString(22, __v) }; + { val __v = domainApiUrl; if (__v != "") _output__.writeString(23, __v) }; } def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): code.obp.grpc.metricsstream.api.MetricEvent = { var __url = this.url @@ -140,6 +143,7 @@ final case class MetricEvent( var __authType = this.authType var __certificateTrust = this.certificateTrust var __certificateTrustDetail = this.certificateTrustDetail + var __domainApiUrl = this.domainApiUrl var _done__ = false while (!_done__) { val _tag__ = _input__.readTag() @@ -167,6 +171,7 @@ final case class MetricEvent( case 162 => __authType = _input__.readString() case 170 => __certificateTrust = _input__.readString() case 178 => __certificateTrustDetail = _input__.readString() + case 186 => __domainApiUrl = _input__.readString() case tag => _input__.skipField(tag) } } @@ -192,7 +197,8 @@ final case class MetricEvent( forwardedFor = __forwardedFor, authType = __authType, certificateTrust = __certificateTrust, - certificateTrustDetail = __certificateTrustDetail + certificateTrustDetail = __certificateTrustDetail, + domainApiUrl = __domainApiUrl ) } def withUrl(__v: _root_.scala.Predef.String): MetricEvent = copy(url = __v) @@ -217,6 +223,7 @@ final case class MetricEvent( def withAuthType(__v: _root_.scala.Predef.String): MetricEvent = copy(authType = __v) def withCertificateTrust(__v: _root_.scala.Predef.String): MetricEvent = copy(certificateTrust = __v) def withCertificateTrustDetail(__v: _root_.scala.Predef.String): MetricEvent = copy(certificateTrustDetail = __v) + def withDomainApiUrl(__v: _root_.scala.Predef.String): MetricEvent = copy(domainApiUrl = __v) def getFieldByNumber(__fieldNumber: _root_.scala.Int): scala.Any = { (__fieldNumber: @_root_.scala.unchecked) match { case 1 => { val __t = url; if (__t != "") __t else null } @@ -241,6 +248,7 @@ final case class MetricEvent( case 20 => { val __t = authType; if (__t != "") __t else null } case 21 => { val __t = certificateTrust; if (__t != "") __t else null } case 22 => { val __t = certificateTrustDetail; if (__t != "") __t else null } + case 23 => { val __t = domainApiUrl; if (__t != "") __t else null } } } def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { @@ -268,6 +276,7 @@ final case class MetricEvent( case 20 => _root_.scalapb.descriptors.PString(authType) case 21 => _root_.scalapb.descriptors.PString(certificateTrust) case 22 => _root_.scalapb.descriptors.PString(certificateTrustDetail) + case 23 => _root_.scalapb.descriptors.PString(domainApiUrl) } } def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) @@ -301,7 +310,8 @@ object MetricEvent extends scalapb.GeneratedMessageCompanion[code.obp.grpc.metri __fieldsMap.getOrElse(__fields.get(18), "").asInstanceOf[_root_.scala.Predef.String], __fieldsMap.getOrElse(__fields.get(19), "").asInstanceOf[_root_.scala.Predef.String], __fieldsMap.getOrElse(__fields.get(20), "").asInstanceOf[_root_.scala.Predef.String], - __fieldsMap.getOrElse(__fields.get(21), "").asInstanceOf[_root_.scala.Predef.String] + __fieldsMap.getOrElse(__fields.get(21), "").asInstanceOf[_root_.scala.Predef.String], + __fieldsMap.getOrElse(__fields.get(22), "").asInstanceOf[_root_.scala.Predef.String] ) } implicit def messageReads: _root_.scalapb.descriptors.Reads[code.obp.grpc.metricsstream.api.MetricEvent] = _root_.scalapb.descriptors.Reads{ @@ -329,7 +339,8 @@ object MetricEvent extends scalapb.GeneratedMessageCompanion[code.obp.grpc.metri __fieldsMap.get(scalaDescriptor.findFieldByNumber(19).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), __fieldsMap.get(scalaDescriptor.findFieldByNumber(20).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), __fieldsMap.get(scalaDescriptor.findFieldByNumber(21).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), - __fieldsMap.get(scalaDescriptor.findFieldByNumber(22).get).map(_.as[_root_.scala.Predef.String]).getOrElse("") + __fieldsMap.get(scalaDescriptor.findFieldByNumber(22).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + __fieldsMap.get(scalaDescriptor.findFieldByNumber(23).get).map(_.as[_root_.scala.Predef.String]).getOrElse("") ) case _ => throw new RuntimeException("Expected PMessage") } @@ -362,6 +373,7 @@ object MetricEvent extends scalapb.GeneratedMessageCompanion[code.obp.grpc.metri def authType: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.authType)((c_, f_) => c_.copy(authType = f_)) def certificateTrust: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.certificateTrust)((c_, f_) => c_.copy(certificateTrust = f_)) def certificateTrustDetail: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.certificateTrustDetail)((c_, f_) => c_.copy(certificateTrustDetail = f_)) + def domainApiUrl: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.domainApiUrl)((c_, f_) => c_.copy(domainApiUrl = f_)) } final val URL_FIELD_NUMBER = 1 final val DATE_FIELD_NUMBER = 2 @@ -385,4 +397,5 @@ object MetricEvent extends scalapb.GeneratedMessageCompanion[code.obp.grpc.metri final val AUTH_TYPE_FIELD_NUMBER = 20 final val CERTIFICATE_TRUST_FIELD_NUMBER = 21 final val CERTIFICATE_TRUST_DETAIL_FIELD_NUMBER = 22 + final val DOMAIN_API_URL_FIELD_NUMBER = 23 } diff --git a/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricsStreamProto.scala b/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricsStreamProto.scala index bc21f42f46..9886e4d6ec 100644 --- a/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricsStreamProto.scala +++ b/obp-api/src/main/scala/code/obp/grpc/metricsstream/api/MetricsStreamProto.scala @@ -77,6 +77,7 @@ object MetricsStreamProto { .addField(stringField("auth_type", 20)) .addField(stringField("certificate_trust", 21)) .addField(stringField("certificate_trust_detail", 22)) + .addField(stringField("domain_api_url", 23)) ) // MetricsStreamService .addService(ServiceDescriptorProto.newBuilder() diff --git a/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala b/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala index 745c203070..1fd82f8728 100644 --- a/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala +++ b/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala @@ -256,7 +256,8 @@ object MetricsArchiveScheduler extends MdcLoggable { i.getConsentReferenceId(), i.getCertificateTrust(), i.getCertificateTrustDetail(), - i.getAuthType() + i.getAuthType(), + i.getDomainApiUrl() ) } diff --git a/obp-api/src/test/scala/code/api/util/MetricFilterParamsTest.scala b/obp-api/src/test/scala/code/api/util/MetricFilterParamsTest.scala new file mode 100644 index 0000000000..074bc150b5 --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/MetricFilterParamsTest.scala @@ -0,0 +1,53 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.util + +import code.api.util.APIUtil.HTTPParam +import org.scalatest.{FlatSpec, Matchers} + +/** + * This suite checks that the documented metric filters reach a query. A filter name is mapped to its query + * param in one place and read in another (APIUtil.createQueriesByHttpParams); a name mapped but not read was + * silently ignored, which is how consent_reference_id and certificate_trust went unapplied. + */ +class MetricFilterParamsTest extends FlatSpec with Matchers { + + private def queries(name: String, value: String): List[OBPQueryParam] = + APIUtil.createQueriesByHttpParams(List(HTTPParam(name, List(value)))).openOrThrowException("filters") + + "createQueriesByHttpParams" should "read consent_reference_id" in { + queries("consent_reference_id", "fd13b9af") should contain(OBPConsentReferenceId("fd13b9af")) + } + + it should "read certificate_trust" in { + queries("certificate_trust", "forwarded") should contain(OBPCertificateTrust("forwarded")) + } + + it should "read domain_api_url" in { + queries("domain_api_url", "/carbon-registry/v1/") should contain(OBPDomainApiUrl("/carbon-registry/v1/")) + } +} diff --git a/obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala index a9b9e22fee..6790852f3d 100644 --- a/obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala +++ b/obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala @@ -219,6 +219,18 @@ class DomainApisTest extends V600ServerSetup { withClue(query.body) { query.code should equal(200) } valuesOf(query.body \ "names", "name") should contain("tree planting") + And("each call's API Metric keeps the OBP URL and records the called one in domain_api_url") + code.metrics.MetricBatchWriter.flush() + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, code.api.util.ApiRole.canReadMetrics.toString) + val metrics = makeGetRequest((baseRequest / "obp" / "v6.0.0" / "management" / "metrics").GET <@ (user1) < s"/$basePath/", "limit" -> "50")) + withClue(metrics.body) { metrics.code should equal(200) } + val rows = (metrics.body \ "metrics").children + val called = rows.map(row => ((row \ "domain_api_url").extract[String], (row \ "url").extract[String])) + called should contain((s"/$basePath/$entity", s"/obp/v7.0.0/banks/$SYS/dynamic-entities/$entity")) + called should contain((s"/$basePath/$queryPath/names", s"/obp/dynamic-endpoint/banks/$SYS/dynamic-resource-doc/$queryPath/names")) + called.foreach { case (domainApiUrl, _) => domainApiUrl should startWith(s"/$basePath/") } + And("a path the space does not serve is a 404") makeGetRequest((under(basePath) / s"nothing_$suffix").GET <@ (user1)).code should equal(404) diff --git a/obp-api/src/test/scala/code/obp/grpc/metricsstream/MetricEventFieldsTest.scala b/obp-api/src/test/scala/code/obp/grpc/metricsstream/MetricEventFieldsTest.scala index 6de7ed8f05..60627c534f 100644 --- a/obp-api/src/test/scala/code/obp/grpc/metricsstream/MetricEventFieldsTest.scala +++ b/obp-api/src/test/scala/code/obp/grpc/metricsstream/MetricEventFieldsTest.scala @@ -18,7 +18,8 @@ class MetricEventFieldsTest extends ServerSetup { 19 -> "forwarded_for", 20 -> "auth_type", 21 -> "certificate_trust", - 22 -> "certificate_trust_detail" + 22 -> "certificate_trust_detail", + 23 -> "domain_api_url" ) private val event = MetricEvent( @@ -28,10 +29,11 @@ class MetricEventFieldsTest extends ServerSetup { forwardedFor = "203.0.113.9, 10.0.0.2, 10.0.0.3", authType = "OAuth2", certificateTrust = "forwarded", - certificateTrustDetail = "CN=proxy,O=Example" + certificateTrustDetail = "CN=proxy,O=Example", + domainApiUrl = "/carbon-registry/v1/activity?limit=10" ) - feature("MetricEvent fields 19 to 22") { + feature("MetricEvent fields 19 to 23") { scenario("the descriptor names them with the numbers the proto file gives them") { val descriptor = MetricsStreamProto.javaDescriptor.findMessageTypeByName("MetricEvent") @@ -49,18 +51,21 @@ class MetricEventFieldsTest extends ServerSetup { event.getFieldByNumber(20) shouldBe "OAuth2" event.getFieldByNumber(21) shouldBe "forwarded" event.getFieldByNumber(22) shouldBe "CN=proxy,O=Example" + event.getFieldByNumber(23) shouldBe "/carbon-registry/v1/activity?limit=10" } scenario("the published JSON payload is read into them") { val payload = parse( """{"url":"/obp/v6.0.0/banks","source_ip":"203.0.113.9","forwarded_for":"203.0.113.9, 10.0.0.2", - |"auth_type":"Consent","certificate_trust":"direct","certificate_trust_detail":""}""".stripMargin) + |"auth_type":"Consent","certificate_trust":"direct","certificate_trust_detail":"", + |"domain_api_url":"/carbon-registry/v1/activity"}""".stripMargin) val fromPayload = MetricsStreamServiceImpl.jsonToMetricEvent(payload) fromPayload.sourceIp shouldBe "203.0.113.9" fromPayload.forwardedFor shouldBe "203.0.113.9, 10.0.0.2" fromPayload.authType shouldBe "Consent" fromPayload.certificateTrust shouldBe "direct" fromPayload.certificateTrustDetail shouldBe "" + fromPayload.domainApiUrl shouldBe "/carbon-registry/v1/activity" } } } From 56404315b25f4a237376b8471772382a9eb99dc1 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 4 Oct 2026 17:55:40 +0200 Subject: [PATCH 03/16] Update DynamicEntityHelper.scala --- .../entity/helper/DynamicEntityHelper.scala | 54 ++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala index a130e2a7f1..9043c19a11 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala @@ -303,8 +303,60 @@ object DynamicEntityHelper { def operationToResourceDoc: Map[(DynamicEntityOperation, String), ResourceDoc] = docsIn(implementedInApiVersion) + /** + * This cache keeps the ResourceDocs that Dynamic Entities generate, so that a request does not + * rebuild them. + * + * The problem it solves: every Dynamic Entity call finds its own ResourceDoc by looking its operation + * up in [[operationToResourceDoc]] (Http4sDynamicEntity calls it for each request). Without this + * cache that lookup built the docs from scratch: for every entity in every space, one doc per + * operation (get all, get one, create, update, patch, delete, and the my, public and community + * variants), each with its description and example bodies, on every request, only to pick one. + * + * What it keeps: one entry per API version, because the same entities are documented twice. v4.0.0 + * documents the unversioned /obp/dynamic-entity/... URLs, and v7.0.0 documents the + * /obp/v7.0.0/banks/BANK_ID/dynamic-entities/... URLs (see [[v700Doc]]). Each entry holds the built + * docs together with the definitions map they were built from. + * + * How it knows when to rebuild: [[docsIn]] asks for the current [[definitionsMap]] and reuses the + * kept docs only if that is the very same object they were built from (`eq`: the same instance, not + * equal contents). So the docs have no expiry of their own; they follow the definitions map cache. + * definitionsMap returns the same object until its time-to-live runs out + * (dynamicEntity.definitions_map.cache.ttl.seconds) or until [[forgetDefinitions]] runs because a + * definition was created, updated or deleted on this node. Either way a new map object appears, the + * identity check fails, and the docs are rebuilt once. In test mode the time-to-live is 0, so every + * call builds a new map and the docs are never reused: tests always see current docs, and never + * exercise this cache. + * + * The docs are shared, as static ones are. ResourceDoc has mutable fields, such as specifiedUrl. + * When every caller got freshly built docs, writing to one affected nobody else; now every request + * gets the same objects, so a write is seen by every other request. The only writer is the + * resource-docs listing (ResourceDocsAPIMethods), which sets specifiedUrl afresh each time. Two + * listings running at once may write the same doc concurrently, which is harmless because a given doc + * always gets the same value: a v7.0.0 doc its v7.0.0 URL, a v4.0.0 doc its dynamic-entity URL. Code + * that sets another field per request, or sets specifiedUrl to a value that depends on the request, + * would leak between requests and must copy the doc first. + * + * Concurrency: two requests that miss at the same moment both build and both store; the last store + * wins, and both are correct. A request still holding an older map may store docs built from it after + * a newer entry; the next caller then fails the identity check and rebuilds, so the stale docs are not + * served for long. + */ + private val docsCache = new java.util.concurrent.ConcurrentHashMap[ScannedApiVersion, (Map[(String, String), DynamicEntityInfo], Map[(DynamicEntityOperation, String), ResourceDoc])]() + /** Every entity's docs in one API version: v4.0.0 for the unversioned URLs, v7.0.0 for the v7.0.0 ones. */ private def docsIn(apiVersion: ScannedApiVersion): Map[(DynamicEntityOperation, String), ResourceDoc] = { + val definitions = definitionsMap + Option(docsCache.get(apiVersion)) match { + case Some((builtFrom, docs)) if builtFrom eq definitions => docs + case _ => + val docs = buildDocsIn(apiVersion, definitions) + docsCache.put(apiVersion, (definitions, docs)) + docs + } + } + + private def buildDocsIn(apiVersion: ScannedApiVersion, definitions: Map[(String, String), DynamicEntityInfo]): Map[(DynamicEntityOperation, String), ResourceDoc] = { val addPrefix = APIUtil.getPropsAsBoolValue("dynamic_entities_have_prefix", true) // record exists tag names, to avoid duplicated dynamic tag name. @@ -345,7 +397,7 @@ object DynamicEntityHelper { ApiTag(tagName) } val fun: DynamicEntityInfo => mutable.Map[(DynamicEntityOperation, String), ResourceDoc] = createDocs(apiTag, apiVersion) - val docs: Iterable[((DynamicEntityOperation, String), ResourceDoc)] = definitionsMap.values.flatMap(fun) + val docs: Iterable[((DynamicEntityOperation, String), ResourceDoc)] = definitions.values.flatMap(fun) docs.toMap } From bf6da8eb8b429a274cb10741814c302c3045a056 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 4 Oct 2026 19:26:00 +0200 Subject: [PATCH 04/16] spreading the word about Platform Apps --- obp-api/src/main/scala/code/api/util/Glossary.scala | 2 ++ .../src/main/scala/code/api/v5_1_0/Http4s510.scala | 13 +++++++++++++ .../code/api/v7_0_0/Http4s700PlatformApps.scala | 6 +++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 2c6d459ef2..373aad49ba 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -1527,6 +1527,8 @@ object Glossary extends MdcLoggable { | |There is a one to one relationship between a Consumer and its certificate. i.e. OBP does not (currently) store the history of certificates bound to a Consumer. If a certificate expires, the third party provider (TPP) must generate a new consumer using a new certificate. In this case, related resources such as rate limits and scopes must be copied from the old consumer to the new consumer. In the future, OBP may store multiple certificates for a consumer, but a certificate will always identify only one consumer record. | + |A Consumer for an app the installation runs itself (the Portal, the API Manager, Opey, OBP-Sentinel or a bank's own service) that calls OBP with its own application token needs Roles granted to it as Scopes. An administrator marks it as a Platform App, the app declares the Scopes it needs, and the administrator grants them. See ${getGlossaryItemLink("Platform Apps")} + | """) glossaryItems += GlossaryItem( diff --git a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala index 20fed68ddd..c24ecdf706 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala @@ -674,6 +674,16 @@ object Http4s510 { } } + // For the Create Consumer docs: how an app the installation runs itself gets the Roles its own calls need. + private val platformAppConsumerText = + s"""**If this Consumer is for an app your installation runs itself** (such as the Portal, the API Manager, + |Opey or a monitoring service like OBP-Sentinel) and the app calls OBP with its own application token + |(OAuth2 client credentials, no User), those calls need Roles granted to the Consumer as Scopes. Have an + |administrator mark the Consumer as a Platform App (`POST /obp/v7.0.0/management/platform-apps`); the app + |then declares the Scopes it needs (`PUT /obp/v7.0.0/consumers/current/platform-app`), retrying until it is + |marked, and the administrator grants the missing ones. See ${Glossary.getGlossaryItemLink("Platform Apps")} + |""".stripMargin + resourceDocs += ResourceDoc( implementedInApiVersion, nameOf(createConsumer), @@ -757,6 +767,8 @@ object Http4s510 { | |**Important**: The key and secret are only shown once in the response. Save them securely as they cannot be retrieved later. | + |$platformAppConsumerText + | |${consumerDisabledText()} | |${userAuthenticationMessage(true)} @@ -3070,6 +3082,7 @@ object Http4s510 { "Create a Consumer", s"""Create a Consumer (Authenticated access). | + |$platformAppConsumerText |""", createConsumerRequestJsonV510, consumerJsonV510, diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700PlatformApps.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700PlatformApps.scala index 96a24cba89..bf34621ba9 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700PlatformApps.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700PlatformApps.scala @@ -218,7 +218,11 @@ object Http4s700PlatformApps { case _ => net.liftweb.common.Empty }).map(code.api.util.APIUtil.unboxFullOrFail(_, Some(cc), ApplicationNotIdentified, 401)) consumerId = consumer.consumerId.get - _ <- Helper.booleanToFuture(PlatformAppNotFound, failCode = 404, cc = Some(cc)) { + // Say which Consumer to mark and where to read how: the app's developer may never have heard of Platform Apps. + notMarked = s"$PlatformAppNotFound This Consumer's CONSUMER_ID is $consumerId (name: ${consumer.name.get}). " + + s"See the glossary entry Platform Apps: GET /obp/v7.0.0/api/glossary/Platform%20Apps or " + + s"${Glossary.apiExplorerUrl}/glossary#Platform%20Apps" + _ <- Helper.booleanToFuture(notMarked, failCode = 404, cc = Some(cc)) { provider.getPlatformApp(consumerId).isDefined } scopes = Option(body.required_scopes).getOrElse(Nil) From 7a4a7e77b15988ad2cb1946e77075daf2c97a802 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 4 Oct 2026 19:31:13 +0200 Subject: [PATCH 05/16] Update parity_allowlist.json --- .../parity_allowlist.json | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/scripts/resource_doc_baseline/parity_allowlist.json b/scripts/resource_doc_baseline/parity_allowlist.json index 49573fcf40..fe8ab5142e 100644 --- a/scripts/resource_doc_baseline/parity_allowlist.json +++ b/scripts/resource_doc_baseline/parity_allowlist.json @@ -1067,14 +1067,6 @@ "lift_digest": "6be19ba2409dbba4ae9f6ed783f5a2068645dd189761c83af8cd0d9f107a22f4", "http4s_digest": "2cb204a9cd0b15471f6fdbe274209d3f4cce2a4d016f0fd8526b0b24fa640d8a" }, - { - "version": "v6_0_0", - "endpoint": "getMetrics", - "field": "description", - "reason": "Documents real new filter params (consent_reference_id, certificate_trust) verified present in MappedMetrics.scala.", - "lift_digest": "04c081020abb8a0f0bfe1b9a850c30151dac9a1cd69f4ea1cbb896388b4237f2", - "http4s_digest": "3e6c35656f1605ba23ae225187a4038f9f4ae48195824e5f4166840c325ec81a" - }, { "version": "v6_0_0", "endpoint": "getPersonalDataFieldById", @@ -1770,6 +1762,30 @@ "reason": "Lists DynamicPathAmbiguous (OBP-09032): creating or renaming a Dynamic Entity after the first segment of a Dynamic Resource Doc of its space, or after a reserved segment, is refused with 409, so a Domain API can publish any space.", "lift_digest": "160f3a99ccf321a6af0129ef59ea81b833b7ff8f636a6557c8fa22a69ef726fd", "http4s_digest": "ac8b5ea3a96b50a33022bfe6226e57f996a2d313d7568eafa396273522140a6a" + }, + { + "version": "v6_0_0", + "endpoint": "getMetrics", + "field": "description", + "reason": "Documents real filter params verified present in MappedMetrics.scala: consent_reference_id and certificate_trust, and domain_api_url (the URL a caller used under a Domain API, matched as starts with).", + "lift_digest": "04c081020abb8a0f0bfe1b9a850c30151dac9a1cd69f4ea1cbb896388b4237f2", + "http4s_digest": "dced69f475028b8f030c52d1ef469e632e916c91ae3343c780aa24515381a0c9" + }, + { + "version": "v5_1_0", + "endpoint": "createConsumer", + "field": "description", + "reason": "Adds a paragraph pointing the developer of an app the installation runs itself (Portal, API Manager, Opey, OBP-Sentinel) to Platform Apps, so its Consumer can be granted Roles as Scopes.", + "lift_digest": "7e639520852912f5fbb9938d9d3f65943054eb8a158dfa7105416a43870d3ce4", + "http4s_digest": "983ad12ac6c61fc5637ee32d4fbc95592ed259c2588b8b50feab0376fec554a6" + }, + { + "version": "v5_1_0", + "endpoint": "createMyConsumer", + "field": "description", + "reason": "Adds a paragraph pointing the developer of an app the installation runs itself (Portal, API Manager, Opey, OBP-Sentinel) to Platform Apps, so its Consumer can be granted Roles as Scopes.", + "lift_digest": "bba5099d62c01e4fd03073a2c54ef3f5669d0cb0f756d8c2556dd48c9dc8cbc2", + "http4s_digest": "fb85fa76f84dab2e7a328d65e38b2d1ecf9eabe90ca99efdd7e8d24ff9bcd152" } ] } From fd3245f693434ebd6f6dc63ef86a30fbae08ca2b Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 4 Oct 2026 20:09:00 +0200 Subject: [PATCH 06/16] fix: let MetricBatchWriter.flush finish one flush before the next starts - A test's flush() could find the queue just drained by the background scheduler and return before those rows were inserted, so DomainApisTest (CI shard 6) saw no metric rows. flush() is now synchronized. Requests are unaffected: they only enqueue, which takes no lock. In the server only the scheduler thread calls flush(), so the lock is uncontended. --- .../src/main/scala/code/metrics/MetricBatchWriter.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala b/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala index 012498078c..36dd03f363 100644 --- a/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala +++ b/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala @@ -169,8 +169,13 @@ object MetricBatchWriter extends MdcLoggable { /** * Drain the queue and batch-insert all pending metrics via Doobie. + * + * This is synchronized so that one flush finishes before the next starts. The background scheduler + * and a caller (a test that flushes before reading the metric table) may flush at the same time. + * Without the lock, a caller could find the queue empty because the scheduler had just drained it, + * and return while those rows were still being inserted, so a read straight after saw none of them. */ - private[code] def flush(): Unit = { + private[code] def flush(): Unit = synchronized { val flushStart = System.nanoTime() // Rows taken off the queue by this flush. If the write fails they are lost, so Telemetry counts them. var drainedRows = 0 From 8fa3dc801f15eb923ef9a1e63ff774ebd7956855 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 4 Oct 2026 20:25:30 +0200 Subject: [PATCH 07/16] testfix: enable write_metrics in the DomainApisTest metric scenario The test props set write_metrics=false, so the Domain API calls in "a Dynamic Entity and a Dynamic Query answer under the base path..." wrote no metric rows and the domain_api_url query returned an empty list --- obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala index 6790852f3d..213e6fbace 100644 --- a/obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala +++ b/obp-api/src/test/scala/code/api/v7_0_0/DomainApisTest.scala @@ -191,6 +191,8 @@ class DomainApisTest extends V600ServerSetup { feature("The Domain API front door") { scenario("a Dynamic Entity and a Dynamic Query answer under the base path as at their OBP URLs, without bank_id") { + // The test props set write_metrics=false; the metric assertions below need the calls recorded. + setPropsValues("write_metrics" -> "true") entityWithRecord(None, entity, "tree planting") queryDoc(None, s"/$queryPath/names", entity, s"domainApiSummary$suffix") grantAllAt(SYS) From f7a11c1b1ef36ddacb3fe93778dbe83a70ca3164 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 5 Oct 2026 09:13:33 +0200 Subject: [PATCH 08/16] Aggregate Metrics client / consumer access and related test --- .../scala/code/api/v6_0_0/Http4s600.scala | 6 ++++-- .../api/v6_0_0/AggregateMetricsTest.scala | 20 +++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 95908b6ea4..bcd00b95ce 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -712,7 +712,7 @@ object Http4s600 { // Route: GET /obp/v6.0.0/management/aggregate-metrics lazy val getAggregateMetrics: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ GET -> `prefixPath` / "management" / "aggregate-metrics" => - EndpointHelpers.withUser(req) { (_, cc) => + EndpointHelpers.executeAndRespond(req) { cc => for { httpParams <- NewStyle.function.extractHttpParamsFromUrl(req.uri.renderString) _ <- NewStyle.function.tryons(ExcludeParametersNotSupported, 400, Some(cc)) { @@ -7852,7 +7852,8 @@ object Http4s600 { "Get Aggregate Metrics", s"""Returns aggregate metrics on api usage eg. total count, response time (in ms), etc. | - |require CanReadAggregateMetrics role + |**Who may call it.** A User with the Role CanReadAggregateMetrics, or an application whose Consumer holds it + |as a Scope, such as a monitoring service run as a Platform App (see ${Glossary.getGlossaryItemLink("Platform Apps")}). | |**NOTE: Automatic from_date Default** | @@ -7950,6 +7951,7 @@ object Http4s600 { ), List(apiTagMetric, apiTagAggregateMetrics), Some(canReadAggregateMetrics :: Nil), + authMode = code.api.util.APIUtil.UserOrApplication, http4sPartialFunction = Some(getAggregateMetrics) ) resourceDocs += ResourceDoc( diff --git a/obp-api/src/test/scala/code/api/v6_0_0/AggregateMetricsTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/AggregateMetricsTest.scala index 741e291b5e..aa04c2f83e 100644 --- a/obp-api/src/test/scala/code/api/v6_0_0/AggregateMetricsTest.scala +++ b/obp-api/src/test/scala/code/api/v6_0_0/AggregateMetricsTest.scala @@ -29,9 +29,10 @@ package code.api.v6_0_0 import code.api.util.APIUtil.OAuth._ import code.api.util.ApiRole.CanReadAggregateMetrics -import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, UserHasMissingRoles} +import code.api.util.ErrorMessages.{ApplicationNotIdentified, UserHasMissingRoles} import code.entitlement.Entitlement import code.metrics.MetricBatchWriter +import code.scope.Scope import com.openbankproject.commons.model.ErrorMessage import com.openbankproject.commons.util.ApiVersion import org.scalatest.Tag @@ -58,7 +59,7 @@ class AggregateMetricsTest extends V600ServerSetup { val response = makeGetRequest(request) Then("We should get a 401") response.code should equal(401) - response.body.extract[ErrorMessage].message should equal(AuthenticatedUserIsRequired) + response.body.extract[ErrorMessage].message should equal(ApplicationNotIdentified) } } @@ -124,4 +125,19 @@ class AggregateMetricsTest extends V600ServerSetup { aggregateMetric2.distinct_consent_count shouldBe 0 } } + + feature(s"test $ApiEndpoint1 version $VersionOfApi - Application access with a Scope") { + scenario("A Consumer holding the Role as a Scope may read aggregate metrics without an Entitlement", ApiEndpoint1, VersionOfApi) { + val request = (v6_0_0_Request / "management" / "aggregate-metrics").GET <@ (user2) + Given("user2 holds no CanReadAggregateMetrics Entitlement and testConsumer2 no Scope") + makeGetRequest(request).code should equal(403) + + When("testConsumer2, which user2 signs with, is granted CanReadAggregateMetrics as a Scope") + val granted = Scope.scope.vend.addScope("", testConsumer2.id.get.toString, CanReadAggregateMetrics.toString) + try { + Then("aggregate metrics are returned") + makeGetRequest(request).code should equal(200) + } finally Scope.scope.vend.deleteScope(granted) + } + } } From 26e9f9b3957776835f1efe93a521ee06be36d3b8 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 5 Oct 2026 14:57:28 +0200 Subject: [PATCH 09/16] Adding decimal and boolean to attribute types + parity_allowlist for getAggregateMetrics --- .../code/api/util/AttributeTypeDocs.scala | 72 +++++++ .../scala/code/api/util/ExampleValue.scala | 2 +- .../scala/code/api/v3_1_0/Http4s310.scala | 20 +- .../scala/code/api/v4_0_0/Http4s400.scala | 42 ++-- .../scala/code/api/v5_1_0/Http4s510.scala | 18 +- .../scala/code/api/v6_0_0/Http4s600.scala | 19 +- .../LocalMappedConnectorInternal.scala | 5 +- obp-api/src/main/scala/code/util/Helper.scala | 9 +- .../code/api/util/AttributeTypeDocsTest.scala | 66 ++++++ .../code/util/CurrencyHandlingTest.scala | 192 ++++++++++++++++++ .../commons/model/enums/Enumerations.scala | 24 +++ .../parity_allowlist.json | 4 +- 12 files changed, 413 insertions(+), 60 deletions(-) create mode 100644 obp-api/src/main/scala/code/api/util/AttributeTypeDocs.scala create mode 100644 obp-api/src/test/scala/code/api/util/AttributeTypeDocsTest.scala create mode 100644 obp-api/src/test/scala/code/util/CurrencyHandlingTest.scala diff --git a/obp-api/src/main/scala/code/api/util/AttributeTypeDocs.scala b/obp-api/src/main/scala/code/api/util/AttributeTypeDocs.scala new file mode 100644 index 0000000000..c1c048f00f --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/AttributeTypeDocs.scala @@ -0,0 +1,72 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.api.util + +import com.openbankproject.commons.model.enums.AttributeType + +/** + * This object holds the one description of the values an attribute's `type` field accepts, for + * every kind of attribute (Bank, Customer, Product, Account, Card, Transaction, Transaction Request, + * User, ATM, Counterparty, Regulated Entity) and for Attribute Definitions. + * + * Each kind of attribute has its own enumeration in obp-commons (`ProductAttributeType`, + * `AccountAttributeType` and so on), but they all accept the same names. The error messages and + * ResourceDoc descriptions used to list those names by hand, in more than forty places, and the + * copies had already drifted apart. They now all read from here. `AttributeTypeDocsTest` checks that + * every enumeration still has exactly the names listed in [[examples]]. + * + * An attribute's value is always stored as text; the type says how the value should be read. + */ +object AttributeTypeDocs { + + /** + * One example value for each attribute type, in the order they are presented to callers. + * DECIMAL is an exact decimal number, which is what rates, prices and face values need: a DOUBLE + * is a binary floating point number and cannot represent most decimal fractions exactly. + */ + val examples: List[(AttributeType.Value, String)] = List( + AttributeType.STRING -> "TAX_NUMBER", + AttributeType.INTEGER -> "123", + AttributeType.DOUBLE -> "12.1234", + AttributeType.DECIMAL -> "1234.5678", + AttributeType.BOOLEAN -> "true", + AttributeType.DATE_WITH_DAY -> "2012-04-23" + ) + + /** The accepted types with an example of each, for error messages, e.g. `STRING(TAX_NUMBER), ... and DATE_WITH_DAY(2012-04-23)`. */ + val typesWithExamples: String = joinWithAnd(examples.map { case (attributeType, example) => s"$attributeType($example)" }) + + /** A sentence for ResourceDoc descriptions that says which values the `type` field accepts and when to use DECIMAL. */ + val typeFieldDescription: String = + s"The type field must be one of ${joinWithAnd(examples.map { case (attributeType, _) => s""""$attributeType"""" })}. " + + s"""Use "${AttributeType.DECIMAL}" rather than "${AttributeType.DOUBLE}" for rates, prices and amounts, because it holds the value exactly; """ + + s""""${AttributeType.BOOLEAN}" values are "true" or "false".""" + + private def joinWithAnd(items: List[String]): String = + if (items.size <= 1) items.mkString else items.init.mkString(", ") + " and " + items.last +} diff --git a/obp-api/src/main/scala/code/api/util/ExampleValue.scala b/obp-api/src/main/scala/code/api/util/ExampleValue.scala index 834cf103fc..5d4af8c6c4 100644 --- a/obp-api/src/main/scala/code/api/util/ExampleValue.scala +++ b/obp-api/src/main/scala/code/api/util/ExampleValue.scala @@ -116,7 +116,7 @@ object ExampleValue { lazy val customerAttributeName = ConnectorField("SPECIAL_TAX_NUMBER", s"The Customer Attribute name, eg: SPECIAL_TAX_NUMBER") glossaryItems += makeGlossaryItem("Customer.customerAttributeName", customerAttributeName) - lazy val customerAttributeType = ConnectorField("STRING", s"It can be ${CustomerAttributeType.STRING}, ${CustomerAttributeType.INTEGER}, ${CustomerAttributeType.DOUBLE}, ${CustomerAttributeType.DATE_WITH_DAY}") + lazy val customerAttributeType = ConnectorField("STRING", s"It can be one of ${AttributeTypeDocs.typesWithExamples}") glossaryItems += makeGlossaryItem("Customer.customerAttributeType", customerAttributeType) lazy val customerAttributeValue = ConnectorField("123456789", s"The Customer Attribute value of the current attribute type, eg: 123456789.") diff --git a/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala b/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala index e2db02d7f9..353c69bb58 100644 --- a/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala +++ b/obp-api/src/main/scala/code/api/v3_1_0/Http4s310.scala @@ -41,7 +41,7 @@ import code.api.util.ApiRole._ import code.api.util.ApiTag._ import code.api.util.ErrorMessages._ import code.api.util.CertificateUtil -import code.api.util.{ApiTrigger, Consent, Glossary, SecureRandomUtil} +import code.api.util.{ApiTrigger, AttributeTypeDocs, Consent, Glossary, SecureRandomUtil} import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware import code.api.util.http4s.IdempotencyMiddleware @@ -2400,7 +2400,7 @@ object Http4s310 { (_, _) <- NewStyle.function.getBank(BankId(bankIdStr), Some(cc)) productAttributeType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${ProductAttributeType.DOUBLE}(12.1234), ${ProductAttributeType.STRING}(TAX_NUMBER), ${ProductAttributeType.INTEGER}(123) and ${ProductAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { ProductAttributeType.withName(postedData.`type`) } (productAttribute, _) <- NewStyle.function.createOrUpdateProductAttribute( BankId(bankIdStr), ProductCode(productCodeStr), None, @@ -2433,7 +2433,7 @@ object Http4s310 { |See [FPML](http://www.fpml.org/) for more examples. | | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | | | @@ -2715,7 +2715,7 @@ object Http4s310 { (_, _) <- NewStyle.function.getBank(BankId(bankIdStr), Some(cc)) productAttributeType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${ProductAttributeType.DOUBLE}(12.1234), ${ProductAttributeType.STRING}(TAX_NUMBER), ${ProductAttributeType.INTEGER}(123) and ${ProductAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { ProductAttributeType.withName(postedData.`type`) } (_, _) <- NewStyle.function.getProductAttributeById(productAttributeIdStr, Some(cc)) (productAttribute, _) <- NewStyle.function.createOrUpdateProductAttribute( @@ -3261,7 +3261,7 @@ object Http4s310 { for { accountAttributeType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${AccountAttributeType.DOUBLE}(2012-04-23), ${AccountAttributeType.STRING}(TAX_NUMBER), ${AccountAttributeType.INTEGER}(123) and ${AccountAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { AccountAttributeType.withName(postedData.`type`) } (_, _) <- NewStyle.function.getBank(BankId(bankIdStr), Some(cc)) (_, _) <- NewStyle.function.getBankAccount(BankId(bankIdStr), AccountId(accountIdStr), Some(cc)) @@ -3298,7 +3298,7 @@ object Http4s310 { | |See [FPML](http://www.fpml.org/) for more examples. | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | |${userAuthenticationMessage(true)} | @@ -3321,7 +3321,7 @@ object Http4s310 { _ <- NewStyle.function.hasEntitlement(bankIdStr, user.userId, canUpdateAccountAttribute, Some(cc)) accountAttributeType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${AccountAttributeType.DOUBLE}(2012-04-23), ${AccountAttributeType.STRING}(TAX_NUMBER), ${AccountAttributeType.INTEGER}(123) and ${AccountAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { AccountAttributeType.withName(postedData.`type`) } (_, _) <- NewStyle.function.getBankAccount(BankId(bankIdStr), AccountId(accountIdStr), Some(cc)) (_, _) <- NewStyle.function.getProduct(BankId(bankIdStr), ProductCode(productCodeStr), Some(cc)) @@ -3688,7 +3688,7 @@ object Http4s310 { (_, _) <- NewStyle.function.getPhysicalCardForBank(BankId(bankIdStr), cardIdStr, Some(cc)) cardAttrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${CardAttributeType.DOUBLE}(12.1234), ${CardAttributeType.STRING}(TAX_NUMBER), ${CardAttributeType.INTEGER}(123) and ${CardAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { CardAttributeType.withName(postedData.`type`) } (cardAttribute, _) <- NewStyle.function.createOrUpdateCardAttribute( Some(BankId(bankIdStr)), Some(cardIdStr), None, @@ -3709,7 +3709,7 @@ object Http4s310 { | |Each Card Attribute is linked to its Card by CARD_ID | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | |${userAuthenticationMessage(true)} | @@ -3743,7 +3743,7 @@ object Http4s310 { (_, _) <- NewStyle.function.getCardAttributeById(cardAttributeIdStr, Some(cc)) cardAttrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${CardAttributeType.DOUBLE}(12.1234), ${CardAttributeType.STRING}(TAX_NUMBER), ${CardAttributeType.INTEGER}(123) and ${CardAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { CardAttributeType.withName(postedData.`type`) } (cardAttribute, _) <- NewStyle.function.createOrUpdateCardAttribute( Some(BankId(bankIdStr)), Some(cardIdStr), Some(cardAttributeIdStr), diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index e72f62a5c8..e405fdb17c 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -65,7 +65,7 @@ import code.DynamicEndpoint.DynamicEndpointSwagger import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} import code.api.util.http4s.ResourceDocMiddleware import code.api.util.http4s.IdempotencyMiddleware -import code.api.util.{APIUtil, CallContext, CustomJsonFormats, NewStyle} +import code.api.util.{APIUtil, AttributeTypeDocs, CallContext, CustomJsonFormats, NewStyle} import code.api.v4_0_0.JSONFactory400._ import code.DynamicData.DynamicData import code.api.util.migration.Migration @@ -895,7 +895,7 @@ object Http4s400 { (_, _) <- NewStyle.function.getBank(BankId(bankIdStr), Some(cc)) productAttributeType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${ProductAttributeType.DOUBLE}(12.1234), ${ProductAttributeType.STRING}(TAX_NUMBER), ${ProductAttributeType.INTEGER}(123) and ${ProductAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { ProductAttributeType.withName(postedData.`type`) } (_, _) <- NewStyle.function.getProduct(BankId(bankIdStr), ProductCode(productCodeStr), Some(cc)) (productAttribute, _) <- NewStyle.function.createOrUpdateProductAttribute( @@ -929,7 +929,7 @@ object Http4s400 { |See [FPML](http://www.fpml.org/) for more examples. | | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | | | @@ -954,7 +954,7 @@ object Http4s400 { (_, _) <- NewStyle.function.getBank(BankId(bankIdStr), Some(cc)) productAttributeType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${ProductAttributeType.DOUBLE}(12.1234), ${ProductAttributeType.STRING}(TAX_NUMBER), ${ProductAttributeType.INTEGER}(123) and ${ProductAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { ProductAttributeType.withName(postedData.`type`) } (_, _) <- NewStyle.function.getProductAttributeById(productAttributeIdStr, Some(cc)) (productAttribute, _) <- NewStyle.function.createOrUpdateProductAttribute( @@ -4622,7 +4622,7 @@ object Http4s400 { for { attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${com.openbankproject.commons.model.enums.BankAttributeType.DOUBLE}(12.1234), ${com.openbankproject.commons.model.enums.BankAttributeType.STRING}(TAX_NUMBER), ${com.openbankproject.commons.model.enums.BankAttributeType.INTEGER}(123) and ${com.openbankproject.commons.model.enums.BankAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { com.openbankproject.commons.model.enums.BankAttributeType.withName(postedData.`type`) } @@ -4641,7 +4641,7 @@ object Http4s400 { bankIdStr, user.userId, canUpdateBankAttribute, Some(cc)) attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${com.openbankproject.commons.model.enums.BankAttributeType.DOUBLE}(12.1234), ${com.openbankproject.commons.model.enums.BankAttributeType.STRING}(TAX_NUMBER), ${com.openbankproject.commons.model.enums.BankAttributeType.INTEGER}(123) and ${com.openbankproject.commons.model.enums.BankAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { com.openbankproject.commons.model.enums.BankAttributeType.withName(postedData.`type`) } @@ -4669,7 +4669,7 @@ object Http4s400 { for { attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${com.openbankproject.commons.model.enums.CustomerAttributeType.DOUBLE}(12.1234), ${com.openbankproject.commons.model.enums.CustomerAttributeType.STRING}(TAX_NUMBER), ${com.openbankproject.commons.model.enums.CustomerAttributeType.INTEGER}(123) and ${com.openbankproject.commons.model.enums.CustomerAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { com.openbankproject.commons.model.enums.CustomerAttributeType.withName(postedData.`type`) } @@ -4688,7 +4688,7 @@ object Http4s400 { for { attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${com.openbankproject.commons.model.enums.CustomerAttributeType.DOUBLE}(12.1234), ${com.openbankproject.commons.model.enums.CustomerAttributeType.STRING}(TAX_NUMBER), ${com.openbankproject.commons.model.enums.CustomerAttributeType.INTEGER}(123) and ${com.openbankproject.commons.model.enums.CustomerAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { com.openbankproject.commons.model.enums.CustomerAttributeType.withName(postedData.`type`) } @@ -4712,7 +4712,7 @@ object Http4s400 { bank.bankId, AccountId(accountIdStr), TransactionId(transactionIdStr), Some(cc)) attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${com.openbankproject.commons.model.enums.TransactionAttributeType.DOUBLE}(12.1234), ${com.openbankproject.commons.model.enums.TransactionAttributeType.STRING}(TAX_NUMBER), ${com.openbankproject.commons.model.enums.TransactionAttributeType.INTEGER} (123)and ${com.openbankproject.commons.model.enums.TransactionAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { com.openbankproject.commons.model.enums.TransactionAttributeType.withName(postedData.`type`) } @@ -4731,7 +4731,7 @@ object Http4s400 { bank.bankId, AccountId(accountIdStr), TransactionId(transactionIdStr), Some(cc)) attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${com.openbankproject.commons.model.enums.TransactionAttributeType.DOUBLE}(12.1234), ${com.openbankproject.commons.model.enums.TransactionAttributeType.STRING}(TAX_NUMBER), ${com.openbankproject.commons.model.enums.TransactionAttributeType.INTEGER} (123)and ${com.openbankproject.commons.model.enums.TransactionAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { com.openbankproject.commons.model.enums.TransactionAttributeType.withName(postedData.`type`) } @@ -4753,7 +4753,7 @@ object Http4s400 { TransactionRequestId(transactionRequestIdStr), Some(cc)) attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${com.openbankproject.commons.model.enums.TransactionRequestAttributeType.DOUBLE}(12.1234), ${com.openbankproject.commons.model.enums.TransactionRequestAttributeType.STRING}(TAX_NUMBER), ${com.openbankproject.commons.model.enums.TransactionRequestAttributeType.INTEGER}(123) and ${com.openbankproject.commons.model.enums.TransactionRequestAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { com.openbankproject.commons.model.enums.TransactionRequestAttributeType.withName(postedData.attribute_type) } @@ -4772,7 +4772,7 @@ object Http4s400 { TransactionRequestId(transactionRequestIdStr), Some(cc)) attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${com.openbankproject.commons.model.enums.TransactionRequestAttributeType.DOUBLE}(12.1234), ${com.openbankproject.commons.model.enums.TransactionRequestAttributeType.STRING}(TAX_NUMBER), ${com.openbankproject.commons.model.enums.TransactionRequestAttributeType.INTEGER}(123) and ${com.openbankproject.commons.model.enums.TransactionRequestAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { com.openbankproject.commons.model.enums.TransactionRequestAttributeType.withName(postedData.attribute_type) } @@ -4825,7 +4825,7 @@ object Http4s400 { for { attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${com.openbankproject.commons.model.enums.UserAttributeType.DOUBLE}(12.1234), ${com.openbankproject.commons.model.enums.UserAttributeType.STRING}(TAX_NUMBER), ${com.openbankproject.commons.model.enums.UserAttributeType.INTEGER} (123)and ${com.openbankproject.commons.model.enums.UserAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { com.openbankproject.commons.model.enums.UserAttributeType.withName(postedData.`type`) } @@ -4846,7 +4846,7 @@ object Http4s400 { } attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${com.openbankproject.commons.model.enums.UserAttributeType.DOUBLE}(12.1234), ${com.openbankproject.commons.model.enums.UserAttributeType.STRING}(TAX_NUMBER), ${com.openbankproject.commons.model.enums.UserAttributeType.INTEGER} (123)and ${com.openbankproject.commons.model.enums.UserAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { com.openbankproject.commons.model.enums.UserAttributeType.withName(postedData.`type`) } @@ -4880,7 +4880,7 @@ object Http4s400 { |See [FPML](http://www.fpml.org/) for more examples. | | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | | | @@ -4925,7 +4925,7 @@ object Http4s400 { s""" Create Customer Attribute | | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | |${userAuthenticationMessage(true)} | @@ -4966,7 +4966,7 @@ object Http4s400 { "Create Transaction Attribute", s""" Create Transaction Attribute | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | |${userAuthenticationMessage(true)} | @@ -5009,7 +5009,7 @@ object Http4s400 { "Create Transaction Request Attribute", s""" Create Transaction Request Attribute | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | |${userAuthenticationMessage(true)} | @@ -5091,7 +5091,7 @@ object Http4s400 { "Create My Personal User Attribute", s""" Create My Personal User Attribute | - |The `type` field must be one of "STRING", "INTEGER", "DOUBLE" or DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | |${userAuthenticationMessage(true)} | @@ -5112,7 +5112,7 @@ object Http4s400 { "Update My Personal User Attribute", s"""Update My Personal User Attribute for current user by USER_ATTRIBUTE_ID | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | |${userAuthenticationMessage(true)} | @@ -5902,7 +5902,7 @@ object Http4s400 { for { attributeType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${com.openbankproject.commons.model.enums.AttributeType.DOUBLE}(12.1234), ${com.openbankproject.commons.model.enums.AttributeType.STRING}(TAX_NUMBER), ${com.openbankproject.commons.model.enums.AttributeType.INTEGER} (123)and ${com.openbankproject.commons.model.enums.AttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { com.openbankproject.commons.model.enums.AttributeType.withName(postedData.`type`) } diff --git a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala index c24ecdf706..23e0aecf70 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala @@ -45,7 +45,7 @@ import code.api.util.newstyle.{BalanceNewStyle, RegulatedEntityAttributeNewStyle import code.api.util.newstyle.RegulatedEntityNewStyle.{createRegulatedEntityNewStyle, deleteRegulatedEntityNewStyle, getRegulatedEntitiesNewStyle, getRegulatedEntityByEntityIdNewStyle} import code.api.util.newstyle.Consumer.createConsumerNewStyle import code.api.util.{APIUtil, Consent, ConsentJWT, CustomJsonFormats, JwtUtil, NewStyle, OBPBankId, OBPLimit, OBPOffset, OBPSortBy, SecureRandomUtil, X509} -import code.api.util.{ExampleValue, Glossary} +import code.api.util.{AttributeTypeDocs, ExampleValue, Glossary} import code.api.v2_0_0.AccountsHelper import code.api.v2_0_0.AccountsHelper.accountTypeFilterText import code.api.berlin.group.v1_3.JSONFactory_BERLIN_GROUP_1_3.{ @@ -1418,7 +1418,7 @@ object Http4s510 { } attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${AtmAttributeType.DOUBLE}(12.1234), ${AtmAttributeType.STRING}(TAX_NUMBER), ${AtmAttributeType.INTEGER}(123) and ${AtmAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { AtmAttributeType.withName(postedData.`type`) } (atmAttribute, _) <- NewStyle.function.createOrUpdateAtmAttribute( bankId, atmId, None, postedData.name, attrType, postedData.value, postedData.is_active, Some(cc)) @@ -1433,7 +1433,7 @@ object Http4s510 { "Create ATM Attribute", s""" Create ATM Attribute | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | |${userAuthenticationMessage(true)} | @@ -1518,7 +1518,7 @@ object Http4s510 { } attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${AtmAttributeType.DOUBLE}(12.1234), ${AtmAttributeType.STRING}(TAX_NUMBER), ${AtmAttributeType.INTEGER}(123) and ${AtmAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { AtmAttributeType.withName(postedData.`type`) } (_, _) <- NewStyle.function.getAtmAttributeById(atmAttributeId, Some(cc)) (atmAttribute, _) <- NewStyle.function.createOrUpdateAtmAttribute( @@ -1722,7 +1722,7 @@ object Http4s510 { } attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${RegulatedEntityAttributeType.DOUBLE}(12.1234), ${RegulatedEntityAttributeType.STRING}(TAX_NUMBER), ${RegulatedEntityAttributeType.INTEGER}(123) and ${RegulatedEntityAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { RegulatedEntityAttributeType.withName(postedData.attribute_type) } (attribute, _) <- RegulatedEntityAttributeNewStyle.createOrUpdateRegulatedEntityAttribute( regulatedEntityId = RegulatedEntityId(entityIdStr), @@ -1742,7 +1742,7 @@ object Http4s510 { s""" | Create a new Regulated Entity Attribute for a given REGULATED_ENTITY_ID. | - | The type field must be one of "STRING", "INTEGER", "DOUBLE" or "DATE_WITH_DAY". + | ${AttributeTypeDocs.typeFieldDescription} | ${userAuthenticationMessage(true)} | """.stripMargin, @@ -1854,7 +1854,7 @@ object Http4s510 { } attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${RegulatedEntityAttributeType.DOUBLE}(12.1234), ${RegulatedEntityAttributeType.STRING}(TAX_NUMBER), ${RegulatedEntityAttributeType.INTEGER}(123) and ${RegulatedEntityAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { RegulatedEntityAttributeType.withName(postedData.attribute_type) } (_, _) <- getRegulatedEntityByEntityIdNewStyle(entityIdStr, Some(cc)) (updated, _) <- RegulatedEntityAttributeNewStyle.createOrUpdateRegulatedEntityAttribute( @@ -2148,7 +2148,7 @@ object Http4s510 { } attrType <- NewStyle.function.tryons( s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${UserAttributeType.DOUBLE}(12.1234), ${UserAttributeType.STRING}(TAX_NUMBER), ${UserAttributeType.INTEGER} (123)and ${UserAttributeType.DATE_WITH_DAY}(2012-04-23)", + AttributeTypeDocs.typesWithExamples, 400, Some(cc)) { UserAttributeType.withName(postedData.`type`) } (userAttribute, _) <- NewStyle.function.createOrUpdateUserAttribute( user.userId, None, postedData.name, attrType, postedData.value, false, Some(cc)) @@ -2163,7 +2163,7 @@ object Http4s510 { "Create Non Personal User Attribute", s""" Create Non Personal User Attribute | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | |${userAuthenticationMessage(true)} | diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index bcd00b95ce..df26694f89 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -49,7 +49,7 @@ import code.api.util.APIUtil.{ urlParametersDocument, userAuthenticationMessage } -import code.api.util.{ExampleValue, Glossary} +import code.api.util.{AttributeTypeDocs, ExampleValue, Glossary} import code.api.v1_2_1.{AccountHolderJSON, BankRoutingJsonV121, TransactionDetailsJSON} import code.api.v4_0_0.BankAttributeBankResponseJsonV400 import code.dynamicchangerequest.MakerChecker @@ -2391,10 +2391,7 @@ object Http4s600 { private val counterpartyAttributeTypeErrorMsg = s"$InvalidJsonFormat The `Type` field can only accept the following field: " + - s"${com.openbankproject.commons.model.enums.CounterpartyAttributeType.DOUBLE}(12.1234), " + - s"${com.openbankproject.commons.model.enums.CounterpartyAttributeType.STRING}(TAX_NUMBER), " + - s"${com.openbankproject.commons.model.enums.CounterpartyAttributeType.INTEGER}(123) and " + - s"${com.openbankproject.commons.model.enums.CounterpartyAttributeType.DATE_WITH_DAY}(2012-04-23)" + AttributeTypeDocs.typesWithExamples // POST /obp/v6.0.0/banks/BANK_ID/accounts/ACCOUNT_ID/counterparties/COUNTERPARTY_ID/attributes (201) lazy val createCounterpartyAttribute: HttpRoutes[IO] = HttpRoutes.of[IO] { @@ -5248,7 +5245,7 @@ object Http4s600 { AbacObjectTypeJsonV600("UserAttributeTrait", "User attribute", List( AbacObjectPropertyJsonV600("name", "String", "Attribute name"), AbacObjectPropertyJsonV600("value", "String", "Attribute value"), - AbacObjectPropertyJsonV600("attributeType", "AttributeType", "Attribute type (STRING, INTEGER, DOUBLE, DATE_WITH_DAY)") + AbacObjectPropertyJsonV600("attributeType", "AttributeType", s"Attribute type (${AttributeTypeDocs.examples.map(_._1).mkString(", ")})") )), AbacObjectTypeJsonV600("AccountAttribute", "Account attribute", List( AbacObjectPropertyJsonV600("name", "String", "Attribute name"), @@ -6094,7 +6091,7 @@ object Http4s600 { // Auth-only; the v6 Lift docs declare `Some(List())` empty role list. private val personalDataTypeErrorMsg = - s"$InvalidJsonFormat The `type` field can only accept: ${UserAttributeType.DOUBLE}, ${UserAttributeType.STRING}, ${UserAttributeType.INTEGER}, ${UserAttributeType.DATE_WITH_DAY}" + s"$InvalidJsonFormat The `type` field can only accept: ${AttributeTypeDocs.typesWithExamples}" // Route: POST /obp/v6.0.0/my/personal-data-fields (201) lazy val createPersonalDataField: HttpRoutes[IO] = HttpRoutes.of[IO] { @@ -9639,7 +9636,7 @@ object Http4s600 { | |For personal attributes that users manage themselves, see the /my/personal-data-fields endpoints. | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or "DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | |${userAuthenticationMessage(true)} |""".stripMargin, @@ -9977,7 +9974,7 @@ object Http4s600 { s""" | Create a new Counterparty Attribute for a given COUNTERPARTY_ID. | - | The type field must be one of "STRING", "INTEGER", "DOUBLE" or "DATE_WITH_DAY". + | ${AttributeTypeDocs.typeFieldDescription} | Authentication is Required | """.stripMargin, @@ -14587,7 +14584,7 @@ object Http4s600 { | |For non-personal attributes that can be used in ABAC rules, see the /users/USER_ID/attributes endpoints. | - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or "DATE_WITH_DAY" + |${AttributeTypeDocs.typeFieldDescription} | |Each Personal Data Field is identified by its own USER_ATTRIBUTE_ID. The "name" is not a unique key: |this endpoint always creates a new field, so the same "name" can occur on multiple fields for the same user @@ -14663,7 +14660,7 @@ object Http4s600 { |USER_ATTRIBUTE_ID identifies the exact field to update; this updates that one field in place and never |creates a new one. The body's "name", "type" and "value" all replace the existing field's values, so a |field can be renamed by changing "name". Returns 404 if no field with that USER_ATTRIBUTE_ID belongs to the user. - |The type field must be one of "STRING", "INTEGER", "DOUBLE" or "DATE_WITH_DAY". + |${AttributeTypeDocs.typeFieldDescription} | |${userAuthenticationMessage(true)} |""".stripMargin, diff --git a/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala b/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala index 88a74c866a..e05e5c28f3 100644 --- a/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala +++ b/obp-api/src/main/scala/code/bankconnectors/LocalMappedConnectorInternal.scala @@ -949,10 +949,7 @@ object LocalMappedConnectorInternal extends MdcLoggable { val attributes = transactionRequestBodyCounterparty.attributes.head val failMsg = s"$InvalidJsonFormat The attribute `type` field can only accept the following field: " + - s"${TransactionRequestAttributeType.DOUBLE}(12.1234)," + - s" ${TransactionRequestAttributeType.STRING}(TAX_NUMBER), " + - s"${TransactionRequestAttributeType.INTEGER}(123) and " + - s"${TransactionRequestAttributeType.DATE_WITH_DAY}(2012-04-23)" + AttributeTypeDocs.typesWithExamples for{ _ <- NewStyle.function.tryons(failMsg, 400, callContext) { diff --git a/obp-api/src/main/scala/code/util/Helper.scala b/obp-api/src/main/scala/code/util/Helper.scala index f1f93ee181..8cfb716a76 100644 --- a/obp-api/src/main/scala/code/util/Helper.scala +++ b/obp-api/src/main/scala/code/util/Helper.scala @@ -174,8 +174,13 @@ object Helper extends Loggable { */ def convertToSmallestCurrencyUnits(amount : BigDecimal, currencyCode : String) : Long = { val decimalPlaces = Helper.currencyDecimalPlaces(currencyCode) - - (amount * BigDecimal("10").pow(decimalPlaces)).toLong + val smallestUnits = (amount * BigDecimal("10").pow(decimalPlaces)).setScale(0, BigDecimal.RoundingMode.DOWN) + // toLong would silently wrap an amount that does not fit in a Long into an unrelated + // (often negative) number, which would then be stored as a balance or transaction amount. + if (!smallestUnits.isValidLong) + throw new IllegalArgumentException( + s"Amount $amount $currencyCode is too large to store: $smallestUnits smallest currency units is outside the range ${Long.MinValue} to ${Long.MaxValue}") + smallestUnits.toLong } diff --git a/obp-api/src/test/scala/code/api/util/AttributeTypeDocsTest.scala b/obp-api/src/test/scala/code/api/util/AttributeTypeDocsTest.scala new file mode 100644 index 0000000000..874a7c7bb7 --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/AttributeTypeDocsTest.scala @@ -0,0 +1,66 @@ +package code.api.util + +import com.openbankproject.commons.model.enums._ +import org.scalatest.{FeatureSpec, GivenWhenThen, Matchers} + +/** + * This class checks that every attribute `type` enumeration accepts the same names, and that the + * shared description in [[AttributeTypeDocs]] lists exactly those names. + * + * Each kind of attribute (Bank, Product, Account and so on) has its own enumeration in obp-commons, + * but error messages and ResourceDoc descriptions for all of them are built from one list. If a + * name were added to one enumeration and not the others, or to the enumerations and not the list, + * callers would be told a type is accepted when it is not, or the reverse. This suite catches that. + * + * Everything here is a pure function call: no server, no database. + */ +class AttributeTypeDocsTest extends FeatureSpec with Matchers with GivenWhenThen { + + private val expectedNames = Set("STRING", "INTEGER", "DOUBLE", "DECIMAL", "BOOLEAN", "DATE_WITH_DAY") + + private val namesPerEnumeration: List[(String, Set[String])] = List( + "AttributeType" -> AttributeType.values.map(_.toString).toSet, + "UserAttributeType" -> UserAttributeType.values.map(_.toString).toSet, + "AtmAttributeType" -> AtmAttributeType.values.map(_.toString).toSet, + "RegulatedEntityAttributeType" -> RegulatedEntityAttributeType.values.map(_.toString).toSet, + "CounterpartyAttributeType" -> CounterpartyAttributeType.values.map(_.toString).toSet, + "BankAttributeType" -> BankAttributeType.values.map(_.toString).toSet, + "AccountAttributeType" -> AccountAttributeType.values.map(_.toString).toSet, + "ProductAttributeType" -> ProductAttributeType.values.map(_.toString).toSet, + "CardAttributeType" -> CardAttributeType.values.map(_.toString).toSet, + "CustomerAttributeType" -> CustomerAttributeType.values.map(_.toString).toSet, + "TransactionAttributeType" -> TransactionAttributeType.values.map(_.toString).toSet, + "TransactionRequestAttributeType" -> TransactionRequestAttributeType.values.map(_.toString).toSet + ) + + feature("Every attribute type enumeration accepts the same names") { + namesPerEnumeration.foreach { case (enumerationName, names) => + scenario(s"$enumerationName accepts STRING, INTEGER, DOUBLE, DECIMAL, BOOLEAN and DATE_WITH_DAY") { + names shouldBe expectedNames + } + } + + scenario("DECIMAL and BOOLEAN can be parsed from the name a caller sends") { + ProductAttributeType.withNameOption("DECIMAL") shouldBe Some(ProductAttributeType.DECIMAL) + AccountAttributeType.withNameOption("BOOLEAN") shouldBe Some(AccountAttributeType.BOOLEAN) + AttributeType.withNameOption("DECIMAL") shouldBe Some(AttributeType.DECIMAL) + } + } + + feature("The shared description lists exactly the accepted names") { + scenario("AttributeTypeDocs.examples has one entry for each accepted name") { + AttributeTypeDocs.examples.map(_._1.toString) should contain theSameElementsAs expectedNames + } + + scenario("The error message text names every type with an example") { + AttributeTypeDocs.typesWithExamples shouldBe + "STRING(TAX_NUMBER), INTEGER(123), DOUBLE(12.1234), DECIMAL(1234.5678), BOOLEAN(true) and DATE_WITH_DAY(2012-04-23)" + } + + scenario("The ResourceDoc description names every type") { + expectedNames.foreach { name => + AttributeTypeDocs.typeFieldDescription should include(s""""$name"""") + } + } + } +} diff --git a/obp-api/src/test/scala/code/util/CurrencyHandlingTest.scala b/obp-api/src/test/scala/code/util/CurrencyHandlingTest.scala new file mode 100644 index 0000000000..912fab637f --- /dev/null +++ b/obp-api/src/test/scala/code/util/CurrencyHandlingTest.scala @@ -0,0 +1,192 @@ +package code.util + +import code.api.util.APIUtil +import org.scalatest.{FeatureSpec, GivenWhenThen, Matchers} + +/** + * This class tests how OBP-API treats currency codes: which codes are accepted, how many decimal + * places each one gets, and how amounts are turned into the stored minor-unit Long and back. + * + * The asset registry design (ideas/ASSET_REGISTRY.md) replaces the code behind these answers, and + * this suite is the check that it keeps every correct answer while doing so. + * + * The suite asserts correct behaviour only. Where the code is known to be wrong today, the correct + * expectation is written inside `pendingUntilFixed`: the scenario reports as pending while the + * defect remains, and fails as soon as the code is fixed, so the wrapper has to be removed in the + * same change and the scenario becomes an ordinary assertion. Behaviour whose correct form has not + * been decided yet (for example how excess decimal places should be rejected) is not tested here. + * + * Everything here is a pure function call: no server, no database. + */ +class CurrencyHandlingTest extends FeatureSpec with Matchers with GivenWhenThen { + + /** + * Every distinct code in media/xml/ISOCurrencyCodes.xml at the time of writing (183 codes). + * It is a literal rather than read from the file so that an edit to the file shows up here as + * a failure, instead of silently changing what the registry would seed. + */ + private val codesInXmlFile: List[String] = List( + "AED", "AFN", "ALL", "AMD", "ANG", "AOA", "ARS", "AUD", "AWG", "AZN", "BAM", "BBD", + "BDT", "BGN", "BHD", "BIF", "BMD", "BND", "BOB", "BOV", "BRL", "BSD", "BTN", "BWP", + "BYN", "BZD", "CAD", "CDF", "CHE", "CHF", "CHW", "CLF", "CLP", "CNY", "COP", "COU", + "CRC", "CUC", "CUP", "CVE", "CZK", "DJF", "DKK", "DOP", "DZD", "EGP", "ERN", "ETB", + "ETH", "EUR", "FJD", "FKP", "GBP", "GEL", "GHS", "GIP", "GMD", "GNF", "GTQ", "GYD", + "HKD", "HNL", "HRK", "HTG", "HUF", "IDR", "ILS", "INR", "IQD", "IRR", "ISK", "JMD", + "JOD", "JPY", "KES", "KGS", "KHR", "KMF", "KPW", "KRW", "KWD", "KYD", "KZT", "LAK", + "LBP", "LKR", "LRD", "LSL", "LYD", "MAD", "MDL", "MGA", "MKD", "MMK", "MNT", "MOP", + "MRU", "MUR", "MVR", "MWK", "MXN", "MXV", "MYR", "MZN", "NAD", "NGN", "NIO", "NOK", + "NPR", "NZD", "OMR", "PAB", "PEN", "PGK", "PHP", "PKR", "PLN", "PYG", "QAR", "RON", + "RSD", "RUB", "RWF", "SAR", "SBD", "SCR", "SDG", "SEK", "SGD", "SHP", "SLL", "SOS", + "SRD", "SSP", "STN", "SVC", "SYP", "SZL", "THB", "TJS", "TMT", "TND", "TOP", "TRY", + "TTD", "TWD", "TZS", "UAH", "UGX", "USD", "USN", "UYI", "UYU", "UYW", "UZS", "VES", + "VND", "VUV", "WST", "XAF", "XAG", "XAU", "XBA", "XBB", "XBC", "XBD", "XCD", "XDR", + "XOF", "XPD", "XPF", "XPT", "XSU", "XTS", "XUA", "XXX", "YER", "ZAR", "ZMW", "ZWL", + "ada", "lovelace", "wei" + ) + + /** XBT is not in the file; isValidCurrencyISOCode appends it in code. */ + private val acceptedCodes: List[String] = codesInXmlFile :+ "XBT" + + /** + * The ISO 4217 minor units of every ISO currency in the file, read from its CcyMnrUnts element. + * Codes whose minor unit ISO gives as "N.A." (precious metals, fund and accounting units) and the + * non-ISO crypto codes are left out: their correct precision is an open question in the design. + */ + private lazy val isoMinorUnits: Map[String, Int] = + (APIUtil.CurrencyIsoCodeFromXmlFile \ "CcyTbl" \ "CcyNtry").flatMap { entry => + val code = (entry \ "Ccy").text.trim + val minorUnits = (entry \ "CcyMnrUnts").text.trim + if (code.nonEmpty && minorUnits.forall(_.isDigit) && minorUnits.nonEmpty && !nonIsoCodes.contains(code)) + Some(code -> minorUnits.toInt) + else None + }.toMap + + private val nonIsoCodes = Set("ETH", "ada", "lovelace", "wei") + + /** Codes where OBP's decimal places differ from ISO 4217 today (ideas/ASSET_REGISTRY.md, Background). */ + private val codesWithWrongDecimalPlaces = Set( + "BHD", "IQD", "JOD", "LYD", "TND", + "CLF", "UYW", + "CZK", + "BIF", "CLP", "DJF", "GNF", "ISK", "KMF", "PYG", "RWF", "UGX", "UYI", "VND", "VUV", "XAF", "XOF", "XPF" + ) + + feature("Which currency codes APIUtil.isValidCurrencyISOCode accepts") { + + scenario("Every code in the XML file, plus XBT, is accepted") { + val rejected = acceptedCodes.filterNot(APIUtil.isValidCurrencyISOCode) + rejected shouldBe Nil + } + + scenario("The XML file holds exactly the codes listed in this test") { + val codesReadFromFile = (APIUtil.CurrencyIsoCodeFromXmlFile \ "CcyTbl" \ "CcyNtry" \ "Ccy") + .map(_.text.trim).filter(_.nonEmpty).distinct.sorted.toList + codesReadFromFile shouldBe codesInXmlFile.sorted + } + + scenario("Values that are not currency codes are rejected") { + APIUtil.isValidCurrencyISOCode("") shouldBe false + APIUtil.isValidCurrencyISOCode("EUR USD") shouldBe false + APIUtil.isValidCurrencyISOCode("978") shouldBe false // EUR's ISO numeric code + APIUtil.isValidCurrencyISOCode("USDC") shouldBe false + APIUtil.isValidCurrencyISOCode("MRO") shouldBe false // replaced by MRU in ISO 4217 + } + + scenario("Currency codes are accepted in any letter case") { + pendingUntilFixed { + List("eur", "Eur", "xbt", "eth", "ADA", "LOVELACE", "WEI").filterNot(APIUtil.isValidCurrencyISOCode) shouldBe Nil + } + } + } + + feature("How many decimal places Helper.currencyDecimalPlaces gives each code") { + + scenario("ISO currencies get their ISO 4217 minor units") { + Given(s"the ${isoMinorUnits.size} ISO currencies in the file with a numeric minor unit, less the known defects") + val mismatches = isoMinorUnits.toList.sorted.collect { + case (code, expected) if !codesWithWrongDecimalPlaces.contains(code) && Helper.currencyDecimalPlaces(code) != expected => + s"$code: expected $expected, got ${Helper.currencyDecimalPlaces(code)}" + } + mismatches shouldBe Nil + } + + scenario("Known defects: these ISO currencies should also get their ISO 4217 minor units") { + pendingUntilFixed { + val mismatches = codesWithWrongDecimalPlaces.toList.sorted.collect { + case code if Helper.currencyDecimalPlaces(code) != isoMinorUnits(code) => + s"$code: expected ${isoMinorUnits(code)}, got ${Helper.currencyDecimalPlaces(code)}" + } + mismatches shouldBe Nil + } + } + + scenario("The decimal places of a code do not depend on its letter case") { + pendingUntilFixed { + Helper.currencyDecimalPlaces("jpy") shouldBe 0 + Helper.currencyDecimalPlaces("kwd") shouldBe 3 + } + } + } + + feature("How Helper.convertToSmallestCurrencyUnits turns an amount into the stored Long") { + + scenario("Amounts within the currency's precision convert exactly") { + Helper.convertToSmallestCurrencyUnits(BigDecimal("12.45"), "EUR") shouldBe 1245L + Helper.convertToSmallestCurrencyUnits(BigDecimal("9034"), "JPY") shouldBe 9034L + Helper.convertToSmallestCurrencyUnits(BigDecimal("1.234"), "KWD") shouldBe 1234L + Helper.convertToSmallestCurrencyUnits(BigDecimal("0"), "EUR") shouldBe 0L + Helper.convertToSmallestCurrencyUnits(BigDecimal("-12.45"), "EUR") shouldBe -1245L + } + + scenario("The largest and smallest amounts that fit in a Long convert exactly") { + Helper.convertToSmallestCurrencyUnits(BigDecimal("92233720368547758.07"), "EUR") shouldBe Long.MaxValue + Helper.convertToSmallestCurrencyUnits(BigDecimal("-92233720368547758.08"), "EUR") shouldBe Long.MinValue + } + + scenario("An amount too large for a Long is refused, not wrapped around") { + Given("10^17 EUR, which is 10^19 cents, more than Long.MaxValue (about 9.22 x 10^18)") + an[IllegalArgumentException] should be thrownBy + Helper.convertToSmallestCurrencyUnits(BigDecimal("100000000000000000"), "EUR") + Given("one cent above Long.MaxValue") + an[IllegalArgumentException] should be thrownBy + Helper.convertToSmallestCurrencyUnits(BigDecimal("92233720368547758.08"), "EUR") + Given("one cent below Long.MinValue") + an[IllegalArgumentException] should be thrownBy + Helper.convertToSmallestCurrencyUnits(BigDecimal("-92233720368547758.09"), "EUR") + } + } + + feature("How Helper.smallestCurrencyUnitToBigDecimal turns the stored Long back into an amount") { + + scenario("The scale of the result is the currency's decimal places") { + val euros = Helper.smallestCurrencyUnitToBigDecimal(1245L, "EUR") + euros shouldBe BigDecimal("12.45") + euros.scale shouldBe 2 + + val yen = Helper.smallestCurrencyUnitToBigDecimal(9034L, "JPY") + yen shouldBe BigDecimal("9034") + yen.scale shouldBe 0 + + val dinars = Helper.smallestCurrencyUnitToBigDecimal(1234L, "KWD") + dinars shouldBe BigDecimal("1.234") + dinars.scale shouldBe 3 + + Helper.smallestCurrencyUnitToBigDecimal(-1245L, "EUR") shouldBe BigDecimal("-12.45") + } + + scenario("Converting an amount within the currency's precision to minor units and back is lossless") { + List(("12.45", "EUR"), ("-0.01", "EUR"), ("99999.99", "EUR"), ("9034", "JPY"), ("1.234", "KWD")).foreach { + case (amount, code) => + val stored = Helper.convertToSmallestCurrencyUnits(BigDecimal(amount), code) + Helper.smallestCurrencyUnitToBigDecimal(stored, code) shouldBe BigDecimal(amount) + } + } + + scenario("The same stored value means a different amount for each precision") { + Given("1000 minor units, which is why a precision change needs a data migration") + Helper.smallestCurrencyUnitToBigDecimal(1000L, "EUR") shouldBe BigDecimal("10.00") + Helper.smallestCurrencyUnitToBigDecimal(1000L, "KWD") shouldBe BigDecimal("1.000") + Helper.smallestCurrencyUnitToBigDecimal(1000L, "JPY") shouldBe BigDecimal("1000") + } + } +} diff --git a/obp-commons/src/main/scala/com/openbankproject/commons/model/enums/Enumerations.scala b/obp-commons/src/main/scala/com/openbankproject/commons/model/enums/Enumerations.scala index 7ec3ed3d24..30a5e255ec 100644 --- a/obp-commons/src/main/scala/com/openbankproject/commons/model/enums/Enumerations.scala +++ b/obp-commons/src/main/scala/com/openbankproject/commons/model/enums/Enumerations.scala @@ -42,6 +42,8 @@ object UserAttributeType extends OBPEnumeration[UserAttributeType]{ object INTEGER extends Value object DOUBLE extends Value object DATE_WITH_DAY extends Value + object DECIMAL extends Value + object BOOLEAN extends Value } sealed trait AtmAttributeType extends EnumValue object AtmAttributeType extends OBPEnumeration[AtmAttributeType]{ @@ -49,6 +51,8 @@ object AtmAttributeType extends OBPEnumeration[AtmAttributeType]{ object INTEGER extends Value object DOUBLE extends Value object DATE_WITH_DAY extends Value + object DECIMAL extends Value + object BOOLEAN extends Value } sealed trait RegulatedEntityAttributeType extends EnumValue object RegulatedEntityAttributeType extends OBPEnumeration[RegulatedEntityAttributeType]{ @@ -56,6 +60,8 @@ object RegulatedEntityAttributeType extends OBPEnumeration[RegulatedEntityAttrib object INTEGER extends Value object DOUBLE extends Value object DATE_WITH_DAY extends Value + object DECIMAL extends Value + object BOOLEAN extends Value } sealed trait CounterpartyAttributeType extends EnumValue object CounterpartyAttributeType extends OBPEnumeration[CounterpartyAttributeType]{ @@ -63,6 +69,8 @@ object CounterpartyAttributeType extends OBPEnumeration[CounterpartyAttributeTyp object INTEGER extends Value object DOUBLE extends Value object DATE_WITH_DAY extends Value + object DECIMAL extends Value + object BOOLEAN extends Value } sealed trait BankAttributeType extends EnumValue object BankAttributeType extends OBPEnumeration[BankAttributeType]{ @@ -70,6 +78,8 @@ object BankAttributeType extends OBPEnumeration[BankAttributeType]{ object INTEGER extends Value object DOUBLE extends Value object DATE_WITH_DAY extends Value + object DECIMAL extends Value + object BOOLEAN extends Value } sealed trait AccountAttributeType extends EnumValue @@ -78,6 +88,8 @@ object AccountAttributeType extends OBPEnumeration[AccountAttributeType]{ object INTEGER extends Value object DOUBLE extends Value object DATE_WITH_DAY extends Value + object DECIMAL extends Value + object BOOLEAN extends Value } sealed trait ProductAttributeType extends EnumValue @@ -86,6 +98,8 @@ object ProductAttributeType extends OBPEnumeration[ProductAttributeType]{ object INTEGER extends Value object DOUBLE extends Value object DATE_WITH_DAY extends Value + object DECIMAL extends Value + object BOOLEAN extends Value } sealed trait CardAttributeType extends EnumValue @@ -94,6 +108,8 @@ object CardAttributeType extends OBPEnumeration[CardAttributeType]{ object INTEGER extends Value object DOUBLE extends Value object DATE_WITH_DAY extends Value + object DECIMAL extends Value + object BOOLEAN extends Value } sealed trait CustomerAttributeType extends EnumValue @@ -102,6 +118,8 @@ object CustomerAttributeType extends OBPEnumeration[CustomerAttributeType]{ object INTEGER extends Value object DOUBLE extends Value object DATE_WITH_DAY extends Value + object DECIMAL extends Value + object BOOLEAN extends Value } sealed trait TransactionAttributeType extends EnumValue @@ -110,6 +128,8 @@ object TransactionAttributeType extends OBPEnumeration[TransactionAttributeType object INTEGER extends Value object DOUBLE extends Value object DATE_WITH_DAY extends Value + object DECIMAL extends Value + object BOOLEAN extends Value } sealed trait TransactionRequestAttributeType extends EnumValue @@ -118,6 +138,8 @@ object TransactionRequestAttributeType extends OBPEnumeration[TransactionReques object INTEGER extends Value object DOUBLE extends Value object DATE_WITH_DAY extends Value + object DECIMAL extends Value + object BOOLEAN extends Value } //------api enumerations ---- @@ -340,6 +362,8 @@ object AttributeType extends OBPEnumeration[AttributeType]{ object INTEGER extends Value object DOUBLE extends Value object DATE_WITH_DAY extends Value + object DECIMAL extends Value + object BOOLEAN extends Value } sealed trait ConsentType extends EnumValue diff --git a/scripts/resource_doc_baseline/parity_allowlist.json b/scripts/resource_doc_baseline/parity_allowlist.json index fe8ab5142e..e0f9a97d2a 100644 --- a/scripts/resource_doc_baseline/parity_allowlist.json +++ b/scripts/resource_doc_baseline/parity_allowlist.json @@ -1007,9 +1007,9 @@ "version": "v6_0_0", "endpoint": "getAggregateMetrics", "field": "description", - "reason": "Documents real new response fields (distinct_user_count etc.) verified present in MappedMetrics.scala.", + "reason": "Documents real new response fields (distinct_user_count etc.) verified present in MappedMetrics.scala, and who may call it: a User with CanReadAggregateMetrics or an application whose Consumer holds it.", "lift_digest": "21e12b117560d354f2ee58bb845e9a5d39d9f47721da8195868cf7a6ed9a216d", - "http4s_digest": "7e83ca3e1b2c02edfe20ecd786d397abb899fe37bc5bf42028fb03734cb627cd" + "http4s_digest": "a1631529945acf190378edeff834bbdef9040aee86cb0282008bb7ee6ac539a8" }, { "version": "v6_0_0", From c78355ba981160dc732539f09778c8ce94821adc Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 5 Oct 2026 16:10:58 +0200 Subject: [PATCH 10/16] Update parity_allowlist.json --- .../parity_allowlist.json | 128 +++++++++++++++++- 1 file changed, 124 insertions(+), 4 deletions(-) diff --git a/scripts/resource_doc_baseline/parity_allowlist.json b/scripts/resource_doc_baseline/parity_allowlist.json index e0f9a97d2a..e81526abd5 100644 --- a/scripts/resource_doc_baseline/parity_allowlist.json +++ b/scripts/resource_doc_baseline/parity_allowlist.json @@ -951,9 +951,9 @@ "version": "v6_0_0", "endpoint": "createPersonalDataField", "field": "description", - "reason": "Additive clarification (USER_ATTRIBUTE_ID identity, non-unique name, PUT-to-update guidance); nothing removed.", + "reason": "Additive clarification (USER_ATTRIBUTE_ID identity, non-unique name, PUT-to-update guidance); nothing removed. Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", "lift_digest": "4cc47e93f3737a605729f4aaf7b439d5ee8e99b737185fe1239487b30980d412", - "http4s_digest": "ae08eb6afbbace39d0834e20610b06ac5d4e50422f84dad91c471a3370267788" + "http4s_digest": "c86601a7e5d2479744e33a91b2d566540935706e61497c12d1444e2fafdfdea9" }, { "version": "v6_0_0", @@ -1159,9 +1159,9 @@ "version": "v6_0_0", "endpoint": "updatePersonalDataField", "field": "description", - "reason": "Additive clarification.", + "reason": "Additive clarification. Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", "lift_digest": "b2a8abcba137c01ec9e67c4c58eb5d7c6596508a12219f87092c5773c29916b3", - "http4s_digest": "85090153d93338f56d26c549900bdba3c2467269c740ef4f9752c7ad88d806dc" + "http4s_digest": "a2b33fe0126d6c9f9891c16de40e54b4f0bf493f9be136c8cd90a5cd99f265ab" }, { "version": "v6_0_0", @@ -1786,6 +1786,126 @@ "reason": "Adds a paragraph pointing the developer of an app the installation runs itself (Portal, API Manager, Opey, OBP-Sentinel) to Platform Apps, so its Consumer can be granted Roles as Scopes.", "lift_digest": "bba5099d62c01e4fd03073a2c54ef3f5669d0cb0f756d8c2556dd48c9dc8cbc2", "http4s_digest": "fb85fa76f84dab2e7a328d65e38b2d1ecf9eabe90ca99efdd7e8d24ff9bcd152" + }, + { + "version": "v3_1_0", + "endpoint": "createAccountAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "ea71aaf30771429dd1dba1a35b08622499620caab6e141361048d8a324bd4f92", + "http4s_digest": "26d7abef135592d998b55bfece976bdfd6747f14308e20d3fccf29409dfc12e3" + }, + { + "version": "v3_1_0", + "endpoint": "createCardAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "7341e2dccb77c534cd3e64c56bbf4e64d00717a575a4850615ef7b4d300a018f", + "http4s_digest": "8649a7fcca672819b3f712bd7387ac3cc25270e89cca5aecf19d68f87e6c3306" + }, + { + "version": "v3_1_0", + "endpoint": "createProductAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "33f42a59104c9bdbf82e51517cd01b20979d532ff0e577fd6cd4d6b8822bf905", + "http4s_digest": "bd886174a46b8ca2c7da5cb67491f74a51a135b983b072e27dbabf16686ed7b8" + }, + { + "version": "v4_0_0", + "endpoint": "createBankAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "cbee89f0568558ddce0d49b2c6f369d2c9add0287fca8e05aa670e4b27c811e2", + "http4s_digest": "512620dbf99d8a55ce360fa89db6cd79c6f688b539203e1e44fcdf30160d6e5f" + }, + { + "version": "v4_0_0", + "endpoint": "createCustomerAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "dfdc89076573354b9884be82fb8db0eae35a40201b9337840f780bc5b0fb3195", + "http4s_digest": "7d27d4c20e94fd14ce92b916eecd567062076f547070059a1713598653fb3ae5" + }, + { + "version": "v4_0_0", + "endpoint": "createMyPersonalUserAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "8681aa4237cc47800556a4e27ddd90c425428cfee130786891f41752a5b975e5", + "http4s_digest": "588ce232ed371a3442cbd6ecdcd07b21e2771650f3261cb5e62bbf964091d379" + }, + { + "version": "v4_0_0", + "endpoint": "createProductAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "33f42a59104c9bdbf82e51517cd01b20979d532ff0e577fd6cd4d6b8822bf905", + "http4s_digest": "bd886174a46b8ca2c7da5cb67491f74a51a135b983b072e27dbabf16686ed7b8" + }, + { + "version": "v4_0_0", + "endpoint": "createTransactionAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "d62af3cc634e1e06bf681dd52cada616e2b94eda105cbb753ca3f48f6a614ad5", + "http4s_digest": "b88a1528bcacfb05158b145be868f65e925221c09c374e3befe872473c170560" + }, + { + "version": "v4_0_0", + "endpoint": "createTransactionRequestAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "fbefcdc67a49bf25ee1dd27cbd12a3bb306a053183be0a951540a1924a377b88", + "http4s_digest": "9c97d6d411b54282b0bc60a2b3079fabf6751b5032187094644cea2db5ba2e18" + }, + { + "version": "v4_0_0", + "endpoint": "updateMyPersonalUserAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "93519b3e3cdcf2e85c316c983d1c80adfe7ebd8efed5dc52a8525c6b1bf21e69", + "http4s_digest": "026008e77d930cb83d2309c5e8fc1de2cd0b12e2b54628b3ff6ffa28aec647bd" + }, + { + "version": "v5_1_0", + "endpoint": "createAtmAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "1b8f1f9a466a4daa57c1729b5b384b0df6d318b5ce201e0337f333e333c202c0", + "http4s_digest": "e93f30b9ba37efbbecd02cfc9580eb481bf6a4628e62554936ad89f69fd6b387" + }, + { + "version": "v5_1_0", + "endpoint": "createNonPersonalUserAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "c887f3d002ffe1214c28ed98a6c5e73a69db00a6e6fb1b7364be3b66eb8c2b38", + "http4s_digest": "c2eefbe2760556dbeaef55e58c8a38be6e5df7f1068a72200bdc43744462944e" + }, + { + "version": "v5_1_0", + "endpoint": "createRegulatedEntityAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "36de16b1bbd7e6f575402e76e6ef58f3beee45529fc5f881dd2684934ca1b309", + "http4s_digest": "a941e5049f4d8f044b450a7d07468a523d1b2c14b93dba4afe3b4794f4e47b5e" + }, + { + "version": "v6_0_0", + "endpoint": "createCounterpartyAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "9b2c5bdc886c56134e569942e28eb9ec39504341ff6968f0e4fd9f3021684d8f", + "http4s_digest": "bb3c66265123795f508806f23a166c9f68aca7f52337598b62cee80c9ab6b7e8" + }, + { + "version": "v6_0_0", + "endpoint": "createUserAttribute", + "field": "description", + "reason": "Lists the accepted attribute types from AttributeTypeDocs.typeFieldDescription, which adds DECIMAL and BOOLEAN (new enum values) to the four the Lift text listed and says when to use DECIMAL.", + "lift_digest": "d2f72f200b8c01cb57f1e07c608a160c125d3e210f9282fbbfa532c72f802248", + "http4s_digest": "789c5598a7af5691282a4192dc5d38199bb068928a392b08314aab3b96d5b6af" } ] } From 2440780abc752c987c1686645326cbafb0335f2e Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 5 Oct 2026 22:39:40 +0200 Subject: [PATCH 11/16] fix: Open Corridor outbox messages go STICKY after a retry limit They were resent for ever and only logged at WARN. After open_corridor.outbox_max_attempts (default 144, about a day) the row goes STICKY for operator reconciliation and is logged at ERROR. Test: OpenCorridorOutboxRetryLimitTest. (sentinel_found) --- .../resources/props/sample.props.template | 7 ++ .../code/api/v7_0_0/JSONFactory7.0.0.scala | 3 +- .../code/messageoutbox/MessageOutbox.scala | 5 +- .../messageoutbox/MessageOutboxRelay.scala | 51 ++++++++++-- .../OpenCorridorOutboxRetryLimitTest.scala | 81 +++++++++++++++++++ 5 files changed, 136 insertions(+), 11 deletions(-) create mode 100644 obp-api/src/test/scala/code/messageoutbox/OpenCorridorOutboxRetryLimitTest.scala diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index 5dcd849c40..c6f144da83 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -1363,6 +1363,13 @@ featured_apis=elasticSearchWarehouseV300 # Only relevant on an instance that has Open Corridor turned on and settles platform fees. # open_corridor.platform_bank_id= +# How many delivery attempts an Open Corridor outbox message gets before the relay stops resending +# it and marks it STICKY for operator reconciliation (GET /management/message-outbox, then its +# /retry). Every attempt that leaves the message undelivered counts: a broker that cannot be reached, +# a retryable error reply, and a settlement instruction whose settlement is not yet FINAL. The wait +# between attempts doubles up to 10 minutes, so the default gives up after roughly a day. +# open_corridor.outbox_max_attempts=144 + # -- Scopes ----------------------------------------------------- # Scopes can be used to limit the APIs a Consumer can call. diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala index 9d4712c53d..5bf00346a6 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala @@ -1679,7 +1679,8 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { * SETTLING / SUBMITTED — the node's last reported rail state (with * `settlement_depth` = confirmation depth when reported) * FINAL — the node reported finality; the instruction row is DELIVERED - * ERROR — the node replied with a non-retryable error (row STICKY); + * ERROR — the node replied with a non-retryable error, or the relay + * gave up after its attempt limit (row STICKY); * operator reconciliation required, see the message's last_error */ case class OpenCorridorSettlementStatusJsonV700( diff --git a/obp-api/src/main/scala/code/messageoutbox/MessageOutbox.scala b/obp-api/src/main/scala/code/messageoutbox/MessageOutbox.scala index a65a666d23..3e94ae4c78 100644 --- a/obp-api/src/main/scala/code/messageoutbox/MessageOutbox.scala +++ b/obp-api/src/main/scala/code/messageoutbox/MessageOutbox.scala @@ -70,8 +70,9 @@ case class OutboxEmailPayload( * Row lifecycle: * PENDING — not yet delivered; the relay keeps publishing with backoff. * DELIVERED — the receiver replied success. - * STICKY — the receiver replied with an error that retrying cannot fix. - * Needs operator reconciliation: visible via + * STICKY — the receiver replied with an error that retrying cannot fix, + * or the relay used up the type's attempt limit without + * delivering it. Needs operator reconciliation: visible via * GET /management/message-outbox, re-queued via its /retry. */ class MessageOutbox extends LongKeyedMapper[MessageOutbox] with IdPK { diff --git a/obp-api/src/main/scala/code/messageoutbox/MessageOutboxRelay.scala b/obp-api/src/main/scala/code/messageoutbox/MessageOutboxRelay.scala index 974c14a632..5befab528c 100644 --- a/obp-api/src/main/scala/code/messageoutbox/MessageOutboxRelay.scala +++ b/obp-api/src/main/scala/code/messageoutbox/MessageOutboxRelay.scala @@ -68,6 +68,12 @@ import scala.concurrent.duration._ * the bank's CBS refusing the credit itself (unknown account, name * mismatch) — the asynchronous beneficiary refusal, distinct from the * transient CBS-DELIVERY-FAILED. + * - Every outcome that keeps a row PENDING counts an attempt. Once a row has + * used open_corridor.outbox_max_attempts of them (a transport failure that + * never clears, a retryable error that keeps coming back, or a settlement + * that never reaches FINAL), it goes STICKY and is logged at ERROR, so an + * operator reconciles it instead of the relay resending it for ever. Its + * last_error keeps the underlying cause. * * EMAIL: each row is first claimed (MessageOutbox.claimForDelivery), so with * several instances only one sends it. Sent → DELIVERED. Not sent → stays @@ -85,6 +91,12 @@ object MessageOutboxRelay extends MdcLoggable { private val perRowTimeout = 60.seconds /** Attempts after which an EMAIL row that could not be sent goes STICKY. */ private val maxEmailAttempts = 8 + /** Attempts after which an OPEN_CORRIDOR row still PENDING goes STICKY. The backoff reaches its + * 10 minute cap after 6 attempts, so the default of 144 gives up after roughly a day. Read on + * every row so that a change to the prop applies without a restart. */ + val defaultMaxOpenCorridorAttempts = 144 + def maxOpenCorridorAttemptsInEffect: Int = + code.api.util.APIUtil.getPropsAsIntValue("open_corridor.outbox_max_attempts", defaultMaxOpenCorridorAttempts) /** A pass can outlast the interval (many emails, or a slow publish); the scheduler would then * start another over the same PENDING rows and deliver them twice. */ private val passRunning = new AtomicBoolean(false) @@ -193,8 +205,9 @@ object MessageOutboxRelay extends MdcLoggable { else "" if (row.operationName == "obp_settlement_instruction" && settlementStatus != "FINAL") { // Broadcast but not final — keep polling by redelivery (§4.4). - row.Attempts(row.attempts + 1).LastError("").LastReplyJson(replyJson).saveMe() - logger.info(s"message outbox row ${row.id.get}: settlement ${row.subjectId} status '$settlementStatus' — will re-poll") + row.LastReplyJson(replyJson) + keepPendingOrGiveUp(row, error = "", cause = s"settlement status '$settlementStatus' is not FINAL", + logRetry = () => logger.info(s"message outbox row ${row.id.get}: settlement ${row.subjectId} status '$settlementStatus' — will re-poll")) } else { row.Status(MessageOutbox.STATUS_DELIVERED).LastError("").LastReplyJson(replyJson).saveMe() logger.info(s"message outbox row ${row.id.get}: ${row.operationName} to ${row.targetId} DELIVERED") @@ -206,18 +219,40 @@ object MessageOutboxRelay extends MdcLoggable { s"STICKY error $errorCode — operator reconciliation required (subject ${row.subjectId})") } else { // Retryable business failure (e.g. SETTLEMENT-FAILED, CBS-DELIVERY-FAILED). - row.Attempts(row.attempts + 1).LastError(errorCode).LastReplyJson(replyJson).saveMe() - logger.warn(s"message outbox row ${row.id.get}: ${row.operationName} to ${row.targetId} " + - s"replied $errorCode — will retry") + row.LastReplyJson(replyJson) + keepPendingOrGiveUp(row, error = errorCode, cause = s"replied $errorCode", + logRetry = () => logger.warn(s"message outbox row ${row.id.get}: ${row.operationName} to ${row.targetId} " + + s"replied $errorCode — will retry")) } case failure => val error = failure match { case Failure(msg, _, _) => msg case _ => "no reply" } - row.Attempts(row.attempts + 1).LastError(error.take(2000)).saveMe() - logger.warn(s"message outbox row ${row.id.get}: ${row.operationName} to ${row.targetId} " + - s"transport failure (attempt ${row.attempts}): $error") + keepPendingOrGiveUp(row, error = error, cause = s"transport failure: $error", + logRetry = () => logger.warn(s"message outbox row ${row.id.get}: ${row.operationName} to ${row.targetId} " + + s"transport failure (attempt ${row.attempts}): $error")) + } + } + + /** + * This method records one more attempt on an OPEN_CORRIDOR row whose outcome would keep it + * PENDING. Below the attempt limit the row stays PENDING and `logRetry` reports the retry. At the + * limit the row goes STICKY instead, with the cause kept in last_error, and is logged at ERROR: + * without a limit a message that can never be delivered is resent every backoff interval for ever + * and never reaches the operator's reconciliation list. + */ + private def keepPendingOrGiveUp(row: MessageOutbox, error: String, cause: String, logRetry: () => Unit): Unit = { + val attempts = row.attempts + 1 + val maxAttempts = maxOpenCorridorAttemptsInEffect + if (attempts >= maxAttempts) { + row.Status(MessageOutbox.STATUS_STICKY).Attempts(attempts) + .LastError(s"gave up after $attempts attempts: $cause".take(2000)).saveMe() + logger.error(s"message outbox row ${row.id.get}: ${row.operationName} to ${row.targetId} " + + s"STICKY after $attempts attempts ($cause) — operator reconciliation required (subject ${row.subjectId})") + } else { + row.Attempts(attempts).LastError(error.take(2000)).saveMe() + logRetry() } } } diff --git a/obp-api/src/test/scala/code/messageoutbox/OpenCorridorOutboxRetryLimitTest.scala b/obp-api/src/test/scala/code/messageoutbox/OpenCorridorOutboxRetryLimitTest.scala new file mode 100644 index 0000000000..15301318eb --- /dev/null +++ b/obp-api/src/test/scala/code/messageoutbox/OpenCorridorOutboxRetryLimitTest.scala @@ -0,0 +1,81 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.messageoutbox + +import code.setup.{DefaultUsers, ServerSetup} +import net.liftweb.mapper.By + +/** + * This test checks that an Open Corridor outbox message the relay can never deliver stops being + * resent once it has used its attempts, and goes STICKY for an operator, as an email does. The + * target bank has no AMQP broker configured, so every attempt is a transport failure. + */ +class OpenCorridorOutboxRetryLimitTest extends ServerSetup with DefaultUsers { + + private val maxAttempts = 3 + + override def beforeEach(): Unit = { + super.beforeEach() + setPropsValues("open_corridor_enabled" -> "true", "open_corridor.outbox_max_attempts" -> maxAttempts.toString) + } + + private def enqueueUndeliverable(): MessageOutbox = + MessageOutbox.enqueue(MessageOutbox.TYPE_OPEN_CORRIDOR, java.util.UUID.randomUUID().toString, + MessageOutbox.SUBJECT_TYPE_SETTLEMENT_ID, "obp_settlement_instruction", + "bank-with-no-broker-" + java.util.UUID.randomUUID().toString, "{}") + + private def reload(row: MessageOutbox): MessageOutbox = + MessageOutbox.find(By(MessageOutbox.id, row.id.get)).openOrThrowException("outbox row") + + feature("Open Corridor outbox messages have a retry limit") { + + scenario("an undeliverable message stays PENDING below the limit and goes STICKY at it") { + val row = enqueueUndeliverable() + + When("the relay fails to deliver it one time fewer than the limit") + (1 until maxAttempts).foreach(_ => MessageOutboxRelay.relayRow(reload(row))) + Then("it is still PENDING, with the transport failure recorded") + val belowLimit = reload(row) + belowLimit.status should equal(MessageOutbox.STATUS_PENDING) + belowLimit.attempts should equal(maxAttempts - 1) + belowLimit.LastError.get should include("BANK_ID") + + When("the relay fails once more") + MessageOutboxRelay.relayRow(reload(row)) + Then("it is STICKY, and last_error says why the relay gave up") + val atLimit = reload(row) + atLimit.status should equal(MessageOutbox.STATUS_STICKY) + atLimit.attempts should equal(maxAttempts) + atLimit.LastError.get should include(s"gave up after $maxAttempts attempts") + atLimit.LastError.get should include("transport failure") + + And("a STICKY row is no longer picked up by the relay") + MessageOutbox.pending().exists(_.id.get == row.id.get) should equal(false) + } + } +} From 39d15b5b7fc6f452e355bf225ad0bd76a217100a Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 5 Oct 2026 23:05:07 +0200 Subject: [PATCH 12/16] fix: a bare obp_exists[X] / obp_not_exists[X] key (no '=') was dropped http4s gives a key with no '=' an empty value list, so the join was silently ignored. It is now a join with no predicate, the same as obp_exists[X]=. Test: JoinQuerySpec. --- .../code/api/dynamic/entity/query/QueryParamParser.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryParamParser.scala b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryParamParser.scala index 028755cd86..fc67a2ded5 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryParamParser.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/query/QueryParamParser.scala @@ -145,9 +145,12 @@ object QueryParamParser { private def parseJoins(params: Map[String, List[String]]): Either[QueryError, List[RawJoin]] = { // NotExistsKey is tried first; `obp_not_exists[...]` never matches `obp_exists[...]` so order is safe either way. + // A bare key (`?obp_exists[CHILD]`, no `=`) arrives with no values; treat it as one empty value + // (no predicate), the same as `?obp_exists[CHILD]=`, rather than silently dropping the join. + def orEmpty(values: List[String]): List[String] = if (values.isEmpty) List("") else values val perKey: List[Either[QueryError, List[RawJoin]]] = params.toList.collect { - case (NotExistsKey(child), values) => traverse(values)(parseOneJoin(Quantifier.NotExists, child, _)) - case (ExistsKey(child), values) => traverse(values)(parseOneJoin(Quantifier.Exists, child, _)) + case (NotExistsKey(child), values) => traverse(orEmpty(values))(parseOneJoin(Quantifier.NotExists, child, _)) + case (ExistsKey(child), values) => traverse(orEmpty(values))(parseOneJoin(Quantifier.Exists, child, _)) } sequence(perKey).map(_.flatten) } From b4ac8c3743d17492fb3d2a623d3b376ef176415a Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 5 Oct 2026 23:11:19 +0200 Subject: [PATCH 13/16] feat: asset registry; currency checks read it Asset tables seeded at boot (ISO codes + XBT, ADA, ETH at today's decimal places). v7.0.0 GET /assets, /assets/ASSET_CODE and /assets/chain/CHAIN_SCHEME/CHAIN_ASSET_ID. isValidCurrencyISOCode and currencyDecimalPlaces now read the registry; answers unchanged except that ADA is accepted as well as ada. Tests: AssetSeedTest, AssetLookupTest, AssetsEndpointTest. --- ideas/ASSET_REGISTRY.md | 363 ++++++++++++++++++ .../main/scala/code/api/util/APIUtil.scala | 16 +- .../src/main/scala/code/api/util/ApiTag.scala | 1 + .../scala/code/api/util/ErrorMessages.scala | 5 + .../main/scala/code/api/util/Glossary.scala | 40 ++ .../scala/code/api/v7_0_0/Http4s700.scala | 11 +- .../code/api/v7_0_0/Http4s700Assets.scala | 203 ++++++++++ .../api/v7_0_0/JSONFactory700Assets.scala | 121 ++++++ obp-api/src/main/scala/code/asset/Asset.scala | 117 ++++++ .../main/scala/code/asset/AssetLookup.scala | 92 +++++ .../src/main/scala/code/asset/AssetSeed.scala | 113 ++++++ .../src/main/scala/code/asset/Assets.scala | 200 ++++++++++ obp-api/src/main/scala/code/util/Helper.scala | 13 +- .../code/api/v7_0_0/AssetsEndpointTest.scala | 180 +++++++++ .../scala/code/asset/AssetLookupTest.scala | 78 ++++ .../test/scala/code/asset/AssetSeedTest.scala | 140 +++++++ .../asset/RestoresSeededAssetRegistry.scala | 20 + .../test/scala/code/setup/ServerSetup.scala | 5 + scripts/asset_registry_currency_report.sql | 114 ++++++ 19 files changed, 1818 insertions(+), 14 deletions(-) create mode 100644 ideas/ASSET_REGISTRY.md create mode 100644 obp-api/src/main/scala/code/api/v7_0_0/Http4s700Assets.scala create mode 100644 obp-api/src/main/scala/code/api/v7_0_0/JSONFactory700Assets.scala create mode 100644 obp-api/src/main/scala/code/asset/Asset.scala create mode 100644 obp-api/src/main/scala/code/asset/AssetLookup.scala create mode 100644 obp-api/src/main/scala/code/asset/AssetSeed.scala create mode 100644 obp-api/src/main/scala/code/asset/Assets.scala create mode 100644 obp-api/src/test/scala/code/api/v7_0_0/AssetsEndpointTest.scala create mode 100644 obp-api/src/test/scala/code/asset/AssetLookupTest.scala create mode 100644 obp-api/src/test/scala/code/asset/AssetSeedTest.scala create mode 100644 obp-api/src/test/scala/code/asset/RestoresSeededAssetRegistry.scala create mode 100644 scripts/asset_registry_currency_report.sql diff --git a/ideas/ASSET_REGISTRY.md b/ideas/ASSET_REGISTRY.md new file mode 100644 index 0000000000..72a3d2617f --- /dev/null +++ b/ideas/ASSET_REGISTRY.md @@ -0,0 +1,363 @@ +# Draft v4: Asset Registry + +**Status**: partly implemented. The registry exists, is seeded, and decides which currency codes are accepted and their decimal places, giving the same answers as the built-in list it replaced. The only changes callers can see so far are two attribute types (`DECIMAL`, `BOOLEAN`) and an amount too large to store being refused instead of wrapped. The progress section below lists what is built and what is next; section 12 lists the open questions. + +**Changes in v4 (2026-10-05)**: adds the progress section. `chain_scheme` names the network as well as the chain (§2); questions 10 and 11 are new. Records what the implementation settled or exposed: the seed has no switch (question 6), Mapper cannot declare the partial unique indexes (§2), the 4–10 character rule for non-seeded codes is not enforced yet (§3), the test setup empties the registry between tests (§11), and the unused `MappedCurrency` table (§11). Code references are brought up to date. + +**Changes in v3 (2026-10-05)**: amounts are stored exactly, as `DECIMAL(38, 18)` in the asset's main unit, instead of as `Long` minor units (§5). The 9-decimal cap and rescaling at the chain boundary are gone: an on-chain amount is always held exactly, never rounded or cut off. Precision corrections become metadata changes instead of rescaling migrations (§7). `ADA` is the single Cardano code at 6 decimals and `ETH` the single Ethereum code at 18; `lovelace` and `wei` stop being currency codes (§7, §10). Currency codes are case-insensitive (§4). Open questions 5, 7 and 8 are settled. + +**Changes in v2**: the precision defect list now comes from the XML rather than memory, and includes currencies whose ISO precision is *lower* than OBP's. The migration handles rescaling downwards. Issuer-less assets are administered at the `SYS` bank instead of through instance-wide roles. Bank-scoped writes check the issuer. Status changes get their own history table. An optional `dti` column is added. A section of implementation notes is added. + +## Background + +OBP-API assumes every `currency` value is an ISO 4217 code with a decimal precision known in code. This blocks accounts denominated in issued assets (tokenised deposits, stablecoins, debt securities, fund shares) and also produces wrong precision for some existing ISO codes. + +Current state: + +| Concern | Where | Behaviour | +|---|---|---| +| Validity of a `currency` value | `APIUtil.isValidCurrencyISOCode` (`code/api/util/APIUtil.scala:847`), 29 call sites | Must appear in `media/xml/ISOCurrencyCodes.xml`, or be `XBT`. Case-sensitive: `eur` is rejected | +| Decimal places | `Helper.currencyDecimalPlaces` (`code/util/Helper.scala:158`) | Hard-coded: CZK/JPY/KRW → 0, KWD/OMR → 3, everything else → 2. Case-sensitive: `jpy` gets 2 | +| Amount → stored value | `Helper.convertToSmallestCurrencyUnits` (`code/util/Helper.scala:175`) | `(amount * 10^dp)`, excess decimals cut off silently. An amount too large for a `Long` is refused (it used to wrap into an unrelated, often negative, number) | +| Stored balances / amounts | `MappedBankAccount.accountBalance`, `MappedTransaction.amount`, `MappedTransaction.newAccountBalance`, standing orders, `bankaccountbalance` | `Long`, minor units | +| Currency column width | `MappedBankAccount.accountCurrency`, `MappedTransaction.currency` | `MappedString(10)` | +| Product fee amount | `ProductFee.Amount` | `MappedDecimal(DECIMAL128, 2)`, fixed 2 decimals regardless of currency | + +`ISOCurrencyCodes.xml` has 283 entries covering 183 distinct codes, and carries `CcyMnrUnts` (the ISO minor-unit count) for every entry. The code does not read it. The file is not pure ISO 4217: it also contains four crypto entries that the Cardano and Ethereum transaction request flows rely on (`LocalMappedConnector.scala:216`, `LocalMappedConnectorInternal.scala:1354`, `:1429`): + +| Code | `CcyMnrUnts` in XML | Note | +|---|---|---| +| `ada` | 6 | Lowercase | +| `lovelace` | 0 | Lowercase, 8 characters | +| `ETH` | 18 | Exact wei amounts do not fit in a `Long` (§5) | +| `wei` | 0 | Lowercase | + +Existing precision defects, independent of tokenisation (ISO value from the XML compared with `currencyDecimalPlaces`): + +| ISO precision vs OBP | Codes | +|---|---| +| ISO 3, OBP 2 | BHD, IQD, JOD, LYD, TND | +| ISO 4, OBP 2 | CLF, UYW | +| ISO 2, OBP 0 | CZK | +| ISO 0, OBP 2 | BIF, CLP, DJF, GNF, ISK, KMF, PYG, RWF, UGX, UYI, VND, VUV, XAF, XOF, XPF | +| ISO `N.A.`, OBP 2 | XAU, XAG, XPT, XPD (precious metals); XDR, XBA, XBB, XBC, XBD, XSU, XUA (fund and accounting units); XTS (testing), XXX (no currency) | +| Not ISO, OBP 2 | XBT (8 in practice), ETH (18), ada (6) | + +MRU and MGA are non-decimal currencies (a `TODO` in `Helper.scala` already notes this); the registry stores them with the precision ISO assigns. + +This document proposes an asset registry that replaces both the ISO check and the hard-coded decimal table, and defines how it relates to Products and Accounts. + +## Progress + +The work is ordered so that each step changes nothing a caller can see until the step that deliberately does, and each answers a question a later step depends on. + +| Step | What | State | +|---|---|---| +| 1 | Tests that describe today's currency behaviour: `CurrencyHandlingTest` (`code.util`) | Done (`26e9f9b39`) | +| 2 | Read-only report of stored amounts per currency code: `scripts/asset_registry_currency_report.sql` | Done, not committed | +| 3 | `DECIMAL` and `BOOLEAN` attribute types (§9) | Done (`26e9f9b39`) | +| 4 | `Asset` and `AssetStatusHistory` tables, provider and boot seed (§2, §6, §7) | Done, not committed | +| 5 | Read-only endpoints `GET /assets`, `GET /assets/ASSET_CODE`, reverse chain lookup (§8), and the Glossary entry (§11) | Done, not committed | +| 6 | `isValidCurrencyISOCode` and `currencyDecimalPlaces` read from the registry (§4) | Done, not committed | +| Later | Case-insensitive codes (§4), rejecting excess decimals (§4), write endpoints (§8), amount storage (§5, §7 Part A), precision corrections (§7 Part B), folding `lovelace` and `wei` (§7 Part C, §10) | Each changes behaviour or waits on an open question | + +**Step 1.** `CurrencyHandlingTest` is a pure unit test (no server, no database). It asserts correct behaviour only. The three known defects are written as the correct expectation inside `pendingUntilFixed`, so they report as pending now and fail, demanding the wrapper's removal, once fixed: +- codes are not accepted in every letter case; +- the codes in the precision table above do not get their ISO minor units; +- `currencyDecimalPlaces` depends on letter case. + +Excess decimals being cut off is not tested, because how they should be rejected (§4) is not decided yet. + +The same change made `convertToSmallestCurrencyUnits` refuse an amount too large for a `Long`. + +**Step 2.** The report runs in one transaction that is made read-only before it reads anything and rolled back at the end. For each currency code and amount column it reports: +- the row count; +- codes that are not upper case; +- rows that would block lowering a precision (question 9); +- `lovelace` and `wei` rows with a fractional amount; +- the largest amount; +- balance records whose account is missing. + +On a local sandbox database every check came back clean: 12 codes in use, the largest amount 40,020,000 SGD. That says little about a real deployment, so it should be run against a copy of a production database before §7 is written. + +**Step 3.** All eleven attribute-type enumerations (`ProductAttributeType`, `AccountAttributeType` and the rest) and `AttributeType`, used by Attribute Definitions, accept `DECIMAL` and `BOOLEAN`. An Attribute Definition can be for any category, so a type only some enumerations accepted would be unusable for the others. The 26 error messages and 17 ResourceDoc descriptions that listed the types by hand now read from one place, `code.api.util.AttributeTypeDocs`; `AttributeTypeDocsTest` checks every enumeration still has exactly those names. As before, a value is stored as text and not checked against its type. + +**Step 4.** `code.asset` holds the `Asset` and `AssetStatusHistory` Mapper entities, the provider `Assets` and the seed `AssetSeed`: +- **`Assets` lookups** find an asset by code, ignoring case. +- **`Assets` writes** refuse a row that breaks the §2 and §3 rules. A code must be 3 to 10 letters or digits, stored upper case. The type and status must be known values. Decimal places must be 0 to 18. Issued types need an issuer, and issuer-less types must not have one. +- **`AssetSeed`** runs at every boot from `Boot.scala`. It inserts the 182 assets of §7 at today's `currencyDecimalPlaces` and never changes an existing row. +- **`AssetSeedTest`** (`code.asset`, CI shard 8) checks the seed against `currencyDecimalPlaces` code by code and checks every rule. + +**Step 5.** `code.api.v7_0_0.Http4s700Assets` serves the three read endpoints of §8, with no authentication and no Role: +- **`GET /assets`** filters by `asset_type`, `status`, `chain_scheme` and `issuer_bank_id`. The first three match ignoring letter case. It pages with `limit` (1 to 500, default 500, enough for every seeded asset) and `offset`, and reports `pagination.total`. A bad value for any of them is refused with 400 (OBP-30581) rather than ignored. +- **`GET /assets/ASSET_CODE`** ignores letter case. An unknown code is 404 (OBP-30579). +- **`GET /assets/chain/CHAIN_SCHEME/CHAIN_ASSET_ID`** matches the scheme ignoring case and the id exactly; no match is 404 (OBP-30580). It finds nothing on a real deployment until the write endpoints can set a chain identity. + +The JSON is in `JSONFactory700Assets.scala`, the endpoints carry the new `Asset` tag, and the Glossary has an "Asset" entry. `AssetsEndpointTest` (CI shard 6) covers each endpoint, the filters, paging, the 400s and the 404s. + +**Step 6.** `code.asset.AssetLookup` answers `APIUtil.isValidCurrencyISOCode` and `Helper.currencyDecimalPlaces` from the registry. The 39 call sites are unchanged. The answers are the ones OBP gave before: +- Codes are matched exactly as written, so `eur` is still refused and `jpy` still gets 2 decimal places. Case-insensitivity remains a later step. +- Status is not checked: a suspended code is still accepted until `isUsableAsset` (§4). +- `ada`, `lovelace` and `wei`, which the built-in list accepts but the registry does not hold under those spellings, stay accepted until §7 Part C. +- A code the registry does not hold gets the built-in decimal places. +- The one visible change: `ADA` in upper case is now accepted, as well as `ada`, because the registry holds it. + +The registry is read once into memory and read again after any write through `Assets`. Every node seeds the same rows, and nothing else writes assets yet, so no cache expiry is needed; the write endpoints (§8) will need one across nodes. If the registry cannot be read (no database) or is empty (before the boot seed), the built-in list answers and nothing is kept. That list survives as `APIUtil.builtInCurrencyCodes` and `Helper.builtInCurrencyDecimalPlaces`; the seed reads it, so an empty database is still seeded at today's precisions. + +`AssetLookupTest` (`code.asset`, CI shard 8) checks that the seeded registry gives the built-in answer for every code, that a newly registered asset is accepted at once with its own decimal places, and that an empty registry falls back. Suites that rewrite the registry restore the seeded one when they end (`RestoresSeededAssetRegistry`), and the test database resets also clear the in-memory copy; otherwise a later suite in the same JVM, such as `CurrencyHandlingTest`, would read a registry holding only test assets. Run with `FundsAvailableTest`, both `TransactionRequestsTest` suites, the v4.0.0 `AccountTest` and `CardanoTransactionRequestTest`: 104 passed, 3 pending (the known defects), none failed. + +--- + +## 1. Model + +Three layers, each with one responsibility: + +| Layer | Scope | Answers | Mutability | +|---|---|---|---| +| **Asset** (new) | Global | What are the units? How many decimals? Who issued them? Which on-chain asset are they? | Identity and precision immutable | +| **Product** (existing) | Bank | What does the bank offer, on what terms? (ISIN, coupon, maturity, documents, fees) | Editable | +| **Account** (existing) | Bank | Who holds how many units, under which product? | Balance changes via transactions | + +Links: + +- `account.currency` → `asset.asset_code` (required; replaces the ISO check). +- `account.product_code` → product at the **account's** bank (existing behaviour; e.g. a custody product at the holder's bank). +- `asset.issuer_bank_id` + `asset.issuer_product_code` → the product at the **issuer's** bank that describes the instrument (optional). + +A holder at bank B holding units issued by bank A has an account at B with `currency = ` and `product_code = `. The instrument terms are found via the asset's issuer product at A, not via the holder's account product. + +**Administering bank.** Every asset is administered at exactly one bank, so every write role can be bank-scoped (in line with retiring any-bank roles). For issued types the administering bank is the issuer, `issuer_bank_id`. Assets with no issuer (`FIAT`, `PRECIOUS_METAL`, `CRYPTO`, and the fund and accounting units) are administered at the `SYS` bank, which is an ordinary bank id. `issuer_bank_id` stays null for them: `SYS` administers the row, it does not issue the asset. Every deployment has a `SYS` bank, so seeding can rely on it being there. + +## 2. `asset` table + +| Column | Type | Required | Mutable | Notes | +|---|---|---|---|---| +| `asset_id` | UUID | yes | no | Internal key | +| `asset_code` | string(10) | yes | no | Unique, ignoring case. The value that appears in `currency` fields. Stored uppercase: `[A-Z0-9]{3,10}`. The legacy lowercase `ada` is seeded as `ADA`; `lovelace` and `wei` are not seeded (§7) | +| `asset_type` | enum | yes | no | See §3 | +| `name` | string(125) | yes | yes | Display name | +| `decimal_places` | int | yes | no (except via the §7 migration) | `0..18`, the asset's true precision; see §5 | +| `issuer_bank_id` | string | for issued types | no | Null for issuer-less types; such assets are administered at `SYS` (§1) | +| `issuer_product_code` | string(50) | no | yes | Product at `issuer_bank_id` describing the instrument | +| `chain_scheme` | string | no | set-once | The chain and network, always both: `CARDANO_MAINNET`, `CARDANO_PREPROD`, `ETHEREUM_SEPOLIA`, … (see below) | +| `chain_asset_id` | string | no | set-once | Cardano: `.`; Ethereum: the token's contract address | +| `dti` | string(9) | no | set-once | ISO 24165 Digital Token Identifier, when one has been assigned | +| `status` | enum | yes | yes | `ACTIVE`, `SUSPENDED`, `RETIRED`, see §6 | +| `created_by_user_id` | string | yes | no | | +| `created_at` / `updated_at` | timestamp | yes | — | | + +Indexes: unique `(asset_code)`; unique `(chain_scheme, chain_asset_id)` where not null; unique `(dti)` where not null; index `(issuer_bank_id)`. + +*As built (step 4)*: the table is `Asset`, with Mapper column names (`AssetCode`, `DecimalPlaces`, `CreationDate`, `LastUpdate` and so on) and a unique index on `AssetId` as well. Optional columns hold an empty string rather than null when unset. The two partial unique indexes are not declared: they must skip unset values, and Mapper can only declare a plain unique index, which would treat every unset (empty) value as a duplicate of every other. Nothing writes these columns yet; the write endpoints (§8) must check uniqueness themselves, or a migration must create the partial indexes in SQL for each database. + +**`chain_scheme` names the network as well as the chain.** The same contract address can exist on Ethereum mainnet, on a test network such as Sepolia, and on networks built on Ethereum such as Polygon or Arbitrum, each time as a different token; Cardano has the same split between mainnet and its preprod and preview test networks. So `(chain_scheme, chain_asset_id)` is only a unique key if the scheme says which network. The form is `_`, upper case, and the network is always written, even for mainnet: OBP does not know which network it is on (it is whatever node `ethereum.rpc.url` or the Cardano wallet API points at), so a bare `CARDANO` would be mainnet on one deployment and preprod on another. A network built on Ethereum is its own chain: `POLYGON_MAINNET`, `ARBITRUM_MAINNET`. The accepted values are a fixed list in code, extended when OBP supports another network. + +This departs from the account routing vocabulary, where a Cardano address uses the bare scheme `CARDANO` (`RoutingSchemeValidation`, `code/routingscheme/RoutingScheme.scala`). Whether account routings should move to the same network-qualified names is question 10. + +`chain_scheme` / `chain_asset_id` are set-once rather than immutable-at-create because a policy id may not exist until the first mint. Once set, they cannot change: reconciliation between OBP balances and on-chain supply depends on the mapping being fixed. `dti` is set-once for the same reason: a DTI is assigned after the token exists. + +## 3. `asset_type` + +| Value | Issuer | Administered at | Created by | +|---|---|---|---| +| `FIAT` | none | `SYS` | Seed only (§7) | +| `PRECIOUS_METAL` | none | `SYS` | Seed only (§7) | +| `ACCOUNTING_UNIT` | none | `SYS` | Seed only (§7): XDR, XBA–XBD, XSU, XUA, XTS, XXX | +| `CRYPTO` | none | `SYS` | Seed, or `CanCreateAsset` at `SYS` | +| `DEPOSIT_TOKEN` | bank | issuer | `CanCreateAsset` at the issuer | +| `STABLECOIN` | bank | issuer | `CanCreateAsset` at the issuer | +| `DEBT_SECURITY` | bank | issuer | `CanCreateAsset` at the issuer | +| `EQUITY` | bank | issuer | `CanCreateAsset` at the issuer | +| `FUND_SHARE` | bank | issuer | `CanCreateAsset` at the issuer | +| `OTHER` | bank | issuer | `CanCreateAsset` at the issuer | + +Codes for non-seeded types must be 4–10 characters, so they cannot collide with a current or future ISO 4217 code. This is not enforced yet: `Assets.createAsset` accepts 3 to 10 characters for every type, because the seed uses it for 3-letter ISO codes. The create endpoint (§8) must enforce it. + +## 4. Validation changes + +- **Currency codes are case-insensitive.** `eur`, `Eur` and `EUR` all name the same asset. Every input is normalised to the stored uppercase form at the API boundary, before validation, comparison or storage, so the rest of the code only ever sees the canonical code. Accepting lowercase without normalising is not enough: `"eur" != "EUR"` in the comparisons transaction requests make against the account's currency, and `currencyDecimalPlaces("jpy")` returns 2 today. Existing rows holding `ada` are rewritten to `ADA`; rows holding `lovelace` or `wei` are converted as described in §7. `FundsAvailableTest` (v3.1.0) currently asserts that `eur` is rejected with 400 and changes with this rule. +- `isValidCurrencyISOCode(code)` → `isUsableAsset(code)`: the asset exists and `status = ACTIVE`. All 29 call sites switch over; the old name stays as a deprecated alias for one release. +- Read paths (GET account, GET transactions) accept any existing asset regardless of status, so suspended or retired holdings remain visible. +- `currencyDecimalPlaces(code)` → `asset.decimal_places`, cached (TTL cache, invalidated on asset write). +- `convertToSmallestCurrencyUnits` already refuses an amount that does not fit in a `Long` (step 1), but it still cuts off excess decimals silently. +- **Amounts are never rounded or cut off.** An amount with more decimal places than the asset's `decimal_places` is rejected with 400. New error code: *invalid amount precision for asset* (number assigned at implementation). An amount that is too large for the storage column (§5) is rejected the same way. +- Transaction requests: `value.currency` must equal the from-account's `currency`. Cross-asset movement goes through an explicit FX/exchange type, never through implicit conversion. +- `fx.scala` fallback rates are unaffected; FX endpoints validate both legs with `isUsableAsset`. + +## 5. Amount storage + +**Rule: OBP holds every amount exactly.** Amounts are money; they are never rounded, cut off or rescaled to fit the storage. + +Today amounts are stored as `Long` minor units (`(amount * 10^dp).toLong`). A `Long` holds at most about 9.22 × 10^18, which cannot hold exact amounts for high-precision assets. At 18 decimal places (ETH, and many Ethereum tokens such as DAI) the largest balance a `Long` can hold is about 9.2 ETH, while total ETH supply is about 1.2 × 10^26 wei. Any cap on `decimal_places` small enough to fit a `Long` means incoming on-chain amounts below the cap cannot be credited exactly, and OBP's ledger stops matching the chain. Rounding hides that; rejecting does not help either, because the funds are already in the bank's wallet. + +So the amount columns change from `Long` minor units to **`DECIMAL(38, 18)` holding the amount in the asset's main unit** (`12.45` EUR, not `1245` cents): + +| Column | Today | After | +|---|---|---| +| `MappedBankAccount.accountBalance` | `MappedLong`, minor units | `DECIMAL(38, 18)`, main unit | +| `MappedTransaction.amount`, `MappedTransaction.newAccountBalance` | `MappedLong`, minor units | same | +| `MappedStandingOrder.AmountValue` | `MappedLong`, minor units | same | +| `BankAccountBalance.BalanceAmount` | `MappedLong`, minor units | same | +| `ProductFee.Amount` | `MappedDecimal(DECIMAL128, 2)` | same | + +- **Why main units.** The stored value no longer depends on `decimal_places`, so correcting an asset's precision (JOD 2 → 3) changes no stored data (§7). With minor units, the same stored number means a different amount at each precision. +- **Why `DECIMAL(38, 18)`.** 38 digits is the largest precision every supported database accepts (SQL Server's maximum). It leaves 20 digits before the decimal point (up to 10^20 − 1 main units) and 18 after, which covers ETH at full precision with total supply to spare. `decimal_places` is therefore capped at 18, the storage scale, and the asset's `decimal_places` (not the column) decides how many of those 18 places an amount may use. +- **The public model does not change.** `BankAccount.balance` and the transaction amount are already `BigDecimal` in `obp-commons` (`BankingModel.scala:213`); only the mapped storage and the two conversion helpers (`convertToSmallestCurrencyUnits`, `smallestCurrencyUnitToBigDecimal`) change, and the helpers can be removed. +- **Connector-sourced accounts** (non-mapped connectors) already report amounts as decimals and are unaffected. + +Recommended `decimal_places`: the asset's real precision. 6 for `ADA` and most Cardano native assets (CIP-68 convention), 18 for `ETH`, 2 for deposit tokens mirroring a 2-decimal fiat, 0 for securities held in whole units. + +## 6. Status + +``` +ACTIVE ⇄ SUSPENDED +ACTIVE → RETIRED +SUSPENDED → RETIRED +``` + +- `ACTIVE`: usable for account creation and transactions. +- `SUSPENDED`: no new accounts, no transaction requests; reads permitted. For regulatory freezes and incident response. Also the way a deployment switches off seeded codes it does not handle (§12, question 6). +- `RETIRED`: terminal. Matured bond, redeemed fund. Reads permitted. + +Every status change writes a row to an `asset_status_history` table (built in step 4 as `AssetStatusHistory`; empty until an endpoint changes a status): `asset_id`, `from_status`, `to_status`, `reason` (free text, required), `changed_by_user_id`, `changed_at`. The metrics trail is not enough for this: metrics can be switched off (`write_metrics`), and they record API calls, not state transitions. A regulator asking "who froze this asset, when, and why" needs the history table. + +## 7. Seeding and migration + +At boot, the registry is seeded idempotently from `ISOCurrencyCodes.xml` (`FIAT`; `PRECIOUS_METAL` for XAU/XAG/XPT/XPD; `ACCOUNTING_UNIT` for the other `N.A.` codes) plus `CRYPTO` for `XBT`, `ADA` and `ETH`. `lovelace` and `wei` are not seeded: they are the smallest units of `ADA` and `ETH`, not separate assets (1 ADA = 10^6 lovelace, 1 ETH = 10^18 wei), and a ledger holds one code per asset. Seeding never overwrites an existing row. + +*Built in step 4* (`code.asset.AssetSeed`): 182 assets, 179 from the file plus `XBT`, `ADA` and `ETH`, each at the precision `currencyDecimalPlaces` gives it today. That is also Part B step 1 below. A code listed for several countries takes the name of its first entry. The seed has no property to switch it off (question 6). If several instances boot at once, the unique index lets one insert win, and the others count the code as already present. + +With amounts stored in main units (§5), the migration has two parts: convert storage once, then correct precisions as metadata. + +**Part A: convert the amount columns.** For each column in the §5 table, in one `runOnce` migration per table, in a single transaction: + +1. Drop any database view that depends on the table's amount columns (see §11). Recreate a view afterwards only if something still needs it. +2. Rename the existing `Long` column to `_legacy_minor_units` (for example `accountbalance_legacy_minor_units`). This is the backup: the original stored values, untouched. +3. Add the new `DECIMAL(38, 18)` column under the original name, and fill it with `legacy_minor_units / 10^legacy_dp`, where `legacy_dp` is what `currencyDecimalPlaces` returns for the row's currency today. This is exact: the stored value was written at that precision, and dividing by a power of ten in decimal arithmetic loses nothing. +4. Verify every row before committing: `new_value * 10^legacy_dp = legacy_minor_units`, and no new value is null where the legacy one was not. Any mismatch rolls the whole table back, and the migration reports it. +5. Write the `legacy_dp` used for each currency into the migration log entry, so the backup can be read back without depending on the old hard-coded table. + +After the migration, application code neither reads nor writes the `_legacy_minor_units` columns; rows created later leave them null. They are kept as a backup until a separate, later migration drops them, once the new columns have been in production long enough to trust (a decision for later, not part of this change). + +**Part B: correct precisions.** After Part A, `decimal_places` only governs validation, so: + +1. Seed each code with the precision OBP uses today (the `currencyDecimalPlaces` table). Behaviour is unchanged. *Done by the step 4 seed.* +2. Where the target precision is **higher** than the legacy one (BHD, IQD, JOD, LYD, TND, CLF, UYW, CZK, XBT → 8, ETH → 18, ADA → 6): update `decimal_places`. No stored data changes; existing amounts simply have trailing zeros. +3. Where the target precision is **lower** (the ISO 0 codes in §Background): update `decimal_places` only if no stored amount in that code has non-zero digits beyond the new precision. Otherwise the code keeps its legacy precision and the migration reports the code and the number of offending rows. It never rounds. +4. `PRECIOUS_METAL` and `ACCOUNTING_UNIT` stay at 2 unless a deployment chooses otherwise, since ISO defines no minor unit. + +**Part C: fold `lovelace` and `wei` into `ADA` and `ETH`.** Rows with currency `lovelace` get currency `ADA` and amount `amount / 10^6`; rows with `wei` get `ETH` and `amount / 10^18`. Both are exact in `DECIMAL(38, 18)`. A `lovelace` or `wei` amount with a fractional part (possible today, because both got 2 decimal places) is not a real on-chain amount: those rows are reported and left unconverted, never rounded. Rows with `ada` become `ADA`. + +Each code and each table is migrated in its own transaction, so one failing check does not hold back the others. + +## 8. Endpoints (v7.0.0) + +All write endpoints are bank-scoped. The bank in the path is the asset's administering bank (§1): the issuer for issued types, `SYS` for issuer-less ones. + +| Method | Path | Role | Notes | +|---|---|---|---| +| GET | `/obp/v7.0.0/assets` | none | Query: `asset_type`, `issuer_bank_id`, `status`, `chain_scheme`, `limit`, `offset`. No authentication | +| GET | `/obp/v7.0.0/assets/ASSET_CODE` | none | No authentication | +| GET | `/obp/v7.0.0/assets/chain/CHAIN_SCHEME/CHAIN_ASSET_ID` | none | Reverse lookup for reconciliation. No authentication | +| POST | `/obp/v7.0.0/banks/BANK_ID/assets` | `CanCreateAsset` | At an issuer: issued types only, `issuer_bank_id` = `BANK_ID`. At `SYS`: `CRYPTO` only, `issuer_bank_id` null | +| PUT | `/obp/v7.0.0/banks/BANK_ID/assets/ASSET_CODE` | `CanUpdateAsset` | Mutable fields only (§2); 400 on any immutable field | +| PUT | `/obp/v7.0.0/banks/BANK_ID/assets/ASSET_CODE/status` | `CanUpdateAssetStatus` | Separate role so freeze authority can be granted alone. Body carries `status` and `reason` (§6) | +| GET | `/obp/v7.0.0/banks/BANK_ID/assets/ASSET_CODE/status-history` | `CanGetAssetStatusHistory` | Rows from `asset_status_history` | +| DELETE | `/obp/v7.0.0/banks/BANK_ID/assets/ASSET_CODE` | `CanDeleteAsset` | Only if no account, transaction, product fee or limit references the code. Seeded assets cannot be deleted | + +**Administering-bank check.** Because `asset_code` is unique across the whole instance but the write paths are scoped to one bank, every PUT and DELETE must check that `BANK_ID` is the asset's administering bank (the issuer, or `SYS` when there is no issuer). Without this, a user holding `CanUpdateAsset` at bank B could change an asset issued by bank A. A mismatch returns 404, so the endpoint does not confirm that the code exists at another bank. + +Seeded assets (`FIAT`, `PRECIOUS_METAL`, `ACCOUNTING_UNIT`, seeded `CRYPTO`) accept `PUT .../status` at `SYS` only. They cannot be created or deleted via the API. + +### POST body + +```json +{ + "asset_code": "TZBOND29", + "asset_type": "DEBT_SECURITY", + "name": "Example Bank 2029 Fixed Rate Note", + "decimal_places": 0, + "issuer_product_code": "TZBOND29", + "chain_scheme": "CARDANO_MAINNET", + "chain_asset_id": ".", + "dti": null +} +``` + +### Response + +```json +{ + "asset_id": "7a1c…", + "asset_code": "TZBOND29", + "asset_type": "DEBT_SECURITY", + "name": "Example Bank 2029 Fixed Rate Note", + "decimal_places": 0, + "issuer_bank_id": "example.bank.tz", + "issuer_product_code": "TZBOND29", + "chain_scheme": "CARDANO_MAINNET", + "chain_asset_id": ".", + "dti": null, + "status": "ACTIVE", + "created_by_user_id": "…", + "created_at": "2026-10-05T10:00:00Z", + "updated_at": "2026-10-05T10:00:00Z" +} +``` + +If `issuer_product_code` is given, the product must exist at `BANK_ID`. + +## 9. Products + +No change to the product model is required. Conventions and fixes: + +**Instrument attributes.** An issuer product describing an asset uses product attributes. Recommended names (convention, not enforced in v1): + +| Name | Type | +|---|---| +| `isin` | STRING | +| `face_value` | DECIMAL | +| `coupon_rate` | DECIMAL | +| `coupon_frequency` | STRING | +| `issue_date` | DATE_WITH_DAY | +| `maturity_date` | DATE_WITH_DAY | +| `terms_sha256` | STRING | + +**Fixes needed:** + +- *Done in step 3*: `DECIMAL` and `BOOLEAN` attribute types, so rates and face values need not be stored as `DOUBLE`. +- `ProductFee.Currency` must pass `isUsableAsset`; `ProductFee.Amount` scale must follow the fee currency's `decimal_places` rather than a fixed 2. + +**Not in v1:** per-category required-attribute schemas (e.g. every `DEBT_SECURITY` issuer product must carry `maturity_date`). See §12. + +## 10. Chain relationship + +The registry stores the on-chain identity of an asset; it does not mint, burn or transfer. On-chain writes belong to the bank's own node holding the bank's keys. OBP-API may read the chain to reconcile `chain_asset_id` supply against the sum of OBP balances in `asset_code`. + +The existing `CARDANO` transaction request body carries `assets: [{policy_id, asset_name, quantity}]` (`LocalMappedConnectorInternal.scala:1360`). With the registry in place, those entries can be resolved to an `asset_code` via the reverse lookup endpoint. + +**Cardano amounts.** The Cardano request carries the amount twice: `value` (OBP currency and amount, today `("lovelace", "1000000")`) and `to.amount` (`quantity` with `unit: "lovelace"`), and nothing checks that they agree. After this change, `value` is in `ADA` (`("ADA", "1")`) and `to.amount` stays in lovelace, because `unit` is Cardano protocol vocabulary and `quantity` is the chain's integer amount. The request is rejected unless `value.amount * 10^6 == to.amount.quantity`. `lovelace` in `value.currency` is rejected with an error naming `ADA`, not silently converted. This changes the v6.0.0 Cardano request in place (v6.0.0 is not yet stable). The same applies to `ETH` and `wei` in the Ethereum request. + +## 11. Implementation notes + +- **Entity naming.** The new Mapper classes must not start with `Mapped` (e.g. `Asset` and `AssetStatusHistory`, not `MappedAsset`), and column objects must not be `m` followed by an uppercase letter. `MappedClassNameTest` enforces this. +- **Documentation.** The feature needs a Glossary entry ("Asset", explaining the three layers and the administering bank) as well as ResourceDocs for each endpoint. Internal notes such as this file do not count. The entry exists since step 5; it says what the registry does not decide yet, and that sentence must change in step 6. +- **SQL on PostgreSQL 16+.** The per-code migrations in §7 build SQL from code values and column names. Test them on PostgreSQL 16 or later, which rejects a bind parameter followed directly by a letter (`$1AND`) that 14 accepts. +- **Views over amount columns.** PostgreSQL refuses to change or rename a column a view depends on, so each Part A migration drops the dependent views first, in the same transaction, and recreates one only if something still needs it. Never drop all views on every boot. Today no OBP view depends on these columns: the only ones that did, `v_fast_firehose_accounts` and `mv_fast_firehose_accounts` (both selecting `mappedbankaccount.accountbalance`), are already dropped by `MigrationOfDropFastFireHoseViews`, and nothing replaces them. The migration should still look up dependent views in the database catalog rather than assume none exist, because a deployment may have created its own; their names belong in the migration log. +- **Tests empty the registry.** The test setup empties every table in `ToSchemify.models` after each test, so the rows the boot seed inserts do not survive into later tests. That is harmless while nothing reads the registry. Step 6 must either exclude `Asset` from that wipe (as `Consumer` and `AuthUser` are) or seed it in test setup, or every currency check in the test suite will fail. +- **`MappedCurrency`.** The existing `code.fx.MappedCurrency` table (code, name, symbol) is never written or read; it is only the foreign-key target of `MappedFXRate`. It is a candidate for removal once FX rates refer to the registry. +- **No silent rounding in Scala.** Scala's `BigDecimal` uses `MathContext.DECIMAL128` by default, which keeps 34 significant digits and rounds arithmetic results beyond that. A `DECIMAL(38, 18)` value can have 38. Amount arithmetic (balance updates, sums, comparisons) must use `MathContext.UNLIMITED`, or check the result is exact; Lift's `MappedDecimal(DECIMAL128, …)` has the same limit and is not suitable for the new columns as is. Add tests at 38 digits. + +## 12. Open questions + +1. **Retirement precondition.** Should `RETIRED` require zero outstanding balance across all banks in this OBP instance? The check is cheap for mapped accounts, impossible for connector-sourced ones. +2. **Multi-instance scope.** `asset_code` is unique per OBP instance. Two OBP instances could register the same code for different assets. Options: prefix issued codes with an issuer identifier, or rely on `chain_asset_id` / ISIN / ISO 24165 DTI as the cross-instance identity and treat `asset_code` as local. *Leaning*: treat `asset_code` as local, and use the chain identity, ISIN or DTI for cross-instance identity. +3. **DTI.** *Leaning, adopted in v2*: the nullable, set-once `dti` column is in §2 now, since it costs little and avoids a later migration. Still open: whether any endpoint should require it. +4. **Required attributes per asset type.** Enforce the §9 attribute set for issued types at product-link time, or leave as convention? +5. **Decimal cap.** *Settled in v3*: amounts are stored exactly as `DECIMAL(38, 18)` (§5), so `decimal_places` is capped at 18 by the storage scale, not by `Long`. Still open: is any planned asset above 18 decimals? NEAR, for example, has 24. Such an asset would need a wider scale, which reduces the digits left before the decimal point. +6. **Seeding vs. config.** Should deployments be able to disable seeded fiat codes (e.g. a node that only handles TZS and one token)? *Adopted in step 4*: the seed has no property; a deployment suspends codes it does not handle via `status = SUSPENDED` at `SYS`, so there is one model and one audit trail. Nothing can suspend a code until the status endpoint (§8) exists. +7. **`ada` and `lovelace`.** *Settled in v3*: one code, `ADA`, at 6 decimal places; `lovelace` remains only as the Cardano protocol unit in the request body (§7 Part C, §10). +8. **`ETH` precision.** *Settled in v3*: `ETH` at its full 18 decimal places, stored exactly (§5); `wei` is folded into `ETH` (§7 Part C). +9. **Codes that fail the downward check.** When a code like ISK keeps legacy precision because some stored amounts have a fractional part, what then? Leave it at 2 permanently, or produce a report so the bank can correct the data and rerun? The step 2 report counts the affected rows; a local sandbox has none, so this needs a run against a copy of real data. +10. **Account routing schemes and networks.** Asset `chain_scheme` values name the network (§2), but a Cardano address on an account still uses the bare routing scheme `CARDANO`, which is mainnet on one deployment and preprod on another. Should account routings move to `CARDANO_MAINNET` and so on? That would change existing routing rows and the Open Corridor settlement account's routing, so it needs its own migration. +11. **Chain identity of native coins.** `ADA` and `ETH` are the chains' own currencies, not tokens, so they have no policy id or contract address. *Leaning*: they have no chain identity; the reverse lookup is for tokens only, and a plain ADA or ETH amount is recognised by the transaction request type. diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 962cfb6c7a..d98e44c76a 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -843,12 +843,16 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ xml } - /** check the currency ISO code from the ISOCurrencyCodes.xml file */ - def isValidCurrencyISOCode(currencyCode: String): Boolean = { - // Note: We add BTC bitcoin as XBT (the ISO compliant varient) - val currencyIsoCodeArray = (CurrencyIsoCodeFromXmlFile \"CcyTbl" \ "CcyNtry" \ "Ccy").map(_.text).mkString(" ").split("\\s+") :+ "XBT" - currencyIsoCodeArray.contains(currencyCode) - } + /** Checks that OBP knows the currency code. The answer comes from the asset registry; see code.asset.AssetLookup. */ + def isValidCurrencyISOCode(currencyCode: String): Boolean = code.asset.AssetLookup.isKnownCode(currencyCode) + + /** + * These are the currency codes OBP accepted before the asset registry existed: every code in the + * ISOCurrencyCodes.xml file, plus XBT (bitcoin under its ISO-style code). AssetLookup falls back to + * them when the registry cannot be read. Use isValidCurrencyISOCode everywhere else. + */ + lazy val builtInCurrencyCodes: Set[String] = + ((CurrencyIsoCodeFromXmlFile \"CcyTbl" \ "CcyNtry" \ "Ccy").map(_.text).mkString(" ").split("\\s+") :+ "XBT").toSet /** Check the id values from GUI, such as ACCOUNT_ID, BANK_ID ... */ def isValidID(id :String):Boolean= { diff --git a/obp-api/src/main/scala/code/api/util/ApiTag.scala b/obp-api/src/main/scala/code/api/util/ApiTag.scala index fe07026af0..adc2db0e02 100644 --- a/obp-api/src/main/scala/code/api/util/ApiTag.scala +++ b/obp-api/src/main/scala/code/api/util/ApiTag.scala @@ -200,6 +200,7 @@ object ApiTag { val apiTagAiAgent = ResourceDocTag("AI-Agent") val apiTagSignalChannel = ResourceDocTag("Signal-Channel") val apiTagFinancialCrime = ResourceDocTag("Financial-Crime") + val apiTagAsset = ResourceDocTag("Asset") private[this] val tagNameSymbolMapTag: MutableMap[String, ResourceDocTag] = MutableMap() diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index 206515e69d..94c1d07426 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -566,6 +566,11 @@ object ErrorMessages { val GlossaryItemShadowsStaticItem = "OBP-30577: A static Glossary Item with this title already exists. Creating this item would override it in the Glossary. Set overrides_static_item to true if that is intended, or choose a different title." val InvalidGlossarySource = "OBP-30578: Invalid source. Please specify all, static or dynamic." + // Asset registry (OBP-30579 .. OBP-30581) + val AssetNotFound = "OBP-30579: Asset not found. Please specify a valid value for ASSET_CODE." + val AssetNotFoundByChainIdentity = "OBP-30580: No asset has this chain identity. Please specify a registered CHAIN_SCHEME and CHAIN_ASSET_ID." + val InvalidAssetQueryParameter = "OBP-30581: Invalid query parameter for assets." + val OrganisationNotFound = "OBP-30506: Organisation not found. Please specify a valid value for ORGANISATION_ID." val OrganisationAlreadyExists = "OBP-30507: Organisation already exists. Please specify a different value for ORGANISATION_ID." val InvalidOrganisationIdFormat = "OBP-30508: Invalid Organisation Id. The ORGANISATION_ID should only contain 0-9/a-z/A-Z/'-'/'.'/'_', and be between 2 and 64 characters in length." diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 373aad49ba..58eca2129b 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -7207,6 +7207,46 @@ object Glossary extends MdcLoggable { """) + glossaryItems += GlossaryItem( + title = "Asset", + description = + s""" + |# Asset + | + |An **Asset** is a unit that amounts can be held in: a currency such as EUR, a precious metal such as gold (XAU), an accounting unit such as the IMF's Special Drawing Right (XDR), a crypto asset such as ETH, or an asset a bank issues, such as a deposit token, a stablecoin, a bond or a fund share. Each Asset has a code, and that code is the value that appears in the `currency` field of an account, a transaction or a product fee. + | + |The **asset registry** lists the Assets an OBP instance knows. It is filled automatically with every currency, metal and accounting unit in ISO 4217 and with the crypto assets XBT, ADA and ETH. Banks will be able to add the assets they issue. + | + |## What the registry records about an Asset + | + |- **Code**: 3 to 10 letters or digits, held upper case. The registry matches codes ignoring letter case, so looking up `eur` finds `EUR`. + |- **Type**: `FIAT`, `PRECIOUS_METAL`, `ACCOUNTING_UNIT`, `CRYPTO`, or one of the types a bank issues: `DEPOSIT_TOKEN`, `STABLECOIN`, `DEBT_SECURITY`, `EQUITY`, `FUND_SHARE`, `OTHER`. + |- **Decimal places**: how many digits an amount may have after the decimal point, from 0 to 18. EUR has 2, JPY has 0. + |- **Issuer**: the bank that issued it, for the types a bank issues. Currencies, metals, accounting units and crypto assets have no issuer. + |- **Chain identity**: for a token recorded on a blockchain, the chain and network (for example `CARDANO_MAINNET`) and the token's identity there. A chain's own currency, such as ADA or ETH, has none. + |- **Status**: `ACTIVE` (usable), `SUSPENDED` (for example frozen by a regulator; existing holdings stay visible) or `RETIRED` (for example a bond that has matured). `RETIRED` is final. + | + |## Assets, Products and Accounts + | + |These are three separate things. The **Asset** says what the units are. A **Product** says what a bank offers and on what terms; an issuing bank describes an instrument, such as a bond's coupon and maturity date, with a Product and its attributes. An **Account** says who holds how many units: its `currency` is the Asset's code, and its product is the holding bank's own Product. A customer at bank B holding a bond issued by bank A has an account at B, in the bond's Asset code, under one of B's Products. + | + |## Administering bank + | + |Every Asset is administered at exactly one bank: the issuer for the types a bank issues, and the `SYS` bank for everything else. Changes to an Asset will be made at its administering bank, so the Roles that allow them can always name one bank. + | + |## Current state + | + |The registry decides which currency codes OBP accepts and how many decimal places it gives them. For now it gives the same answers as the built-in list OBP used before: codes are matched exactly as written, so `EUR` is accepted and `eur` is not, and an Asset's status is not yet checked. Each code has the same number of decimal places as in that list. The registry does not hold `lovelace` or `wei`, which OBP still accepts: they are the smallest units of ADA and ETH, not Assets of their own. OBP also still accepts `ada` as well as `ADA`. + | + |## Endpoints + | + |These need no authentication. + | + |- [Get Assets](${apiExplorerUrl}/resource-docs/OBPv7.0.0?operationid=OBPv7.0.0-getAssets): `GET /obp/v7.0.0/assets` + |- [Get Asset](${apiExplorerUrl}/resource-docs/OBPv7.0.0?operationid=OBPv7.0.0-getAsset): `GET /obp/v7.0.0/assets/ASSET_CODE` + |- [Get Asset by Chain Identity](${apiExplorerUrl}/resource-docs/OBPv7.0.0?operationid=OBPv7.0.0-getAssetByChainIdentity): `GET /obp/v7.0.0/assets/chain/CHAIN_SCHEME/CHAIN_ASSET_ID` + |""".stripMargin) + /////////////////////////////////////////////////////////////////// // NOTE! Some glossary items are generated in ExampleValue.scala ////////////////////////////////////////////////////////////////// diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index 91f369d7d3..9f8d60ff53 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -4836,11 +4836,11 @@ object Http4s700 { "GET", "/management/message-outbox", "Get Message Outbox", - """List rows of the generic transactional message outbox — the messages OBP-API must deliver asynchronously, written in the same DB transaction as the business event that caused them and published by the relay with at-least-once redelivery. + s"""List rows of the generic transactional message outbox — the messages OBP-API must deliver asynchronously, written in the same DB transaction as the business event that caused them and published by the relay with at-least-once redelivery. | |Filter with `outbox_type` (e.g. `OPEN_CORRIDOR`), `status` (`PENDING` / `DELIVERED` / `STICKY`) and `limit` (default 100, max 500). `subject_id` + `subject_id_type` name the business object each message is about (a settlement, a transaction request, ...) — not to be confused with the per-request Correlation-Id. | - |STICKY rows are failures redelivery cannot fix; after reconciliation, re-queue one with the retry endpoint. The wire payload is not exposed: it can carry commit-reveal evidence and originator PII. + |STICKY rows are failures redelivery cannot fix: an error reply that retrying cannot change, or an Open Corridor message still undelivered after ${code.messageoutbox.MessageOutboxRelay.maxOpenCorridorAttemptsInEffect} attempts on this instance (its `last_error` keeps the cause). After reconciliation, re-queue one with the retry endpoint. The wire payload is not exposed: it can carry commit-reveal evidence and originator PII. | |Authentication is Required.""".stripMargin, EmptyBody, @@ -5310,12 +5310,12 @@ object Http4s700 { "GET", "/banks/BANK_ID/open-corridor/settlements/SETTLEMENT_ID", "Get Open Corridor Settlement", - """Read one Open Corridor settlement. BANK_ID must be a party (debtor or creditor) of the settlement — other banks get a 404. + s"""Read one Open Corridor settlement. BANK_ID must be a party (debtor or creditor) of the settlement — other banks get a 404. | |The two status fields deliberately separate the two layers: | |* `ledger_status` — the OBP-side OPEN_CORRIDOR_SETTLEMENT Transaction Request (COMPLETED at settle time: netting, promise discharge and the net ledger Transaction are done). - |* `settlement_status` — the value leg on the rail, as last reported by the debtor bank's node: `NET_ZERO` (nothing to move), `INSTRUCTED` (no node reply yet), `SETTLING` / `SUBMITTED` (in flight, with `settlement_depth` = confirmation depth when reported), `FINAL` (node reported finality), `ERROR` (non-retryable node error; operator reconciliation — see the message's `last_error`). + |* `settlement_status` — the value leg on the rail, as last reported by the debtor bank's node: `NET_ZERO` (nothing to move), `INSTRUCTED` (no node reply yet), `SETTLING` / `SUBMITTED` (in flight, with `settlement_depth` = confirmation depth when reported), `FINAL` (node reported finality), `ERROR` (a non-retryable node error, or the instruction was still not FINAL after ${code.messageoutbox.MessageOutboxRelay.maxOpenCorridorAttemptsInEffect} delivery attempts on this instance; operator reconciliation — see the message's `last_error`). | |`messages` lists the settlement's Interface C outbox rows (settlement advices and the settlement instruction) with their delivery state. | @@ -7473,6 +7473,9 @@ object Http4s700 { // Deployment Checks: is this instance, and what sits in front of it, set up correctly. resourceDocs ++= Http4s700DeploymentChecks.resourceDocs + // The asset registry: the currencies, metals, accounting units, crypto assets and bank-issued assets amounts are held in. + resourceDocs ++= Http4s700Assets.resourceDocs + val allRoutes: HttpRoutes[IO] = { val sorted = resourceDocs .sortBy(rd => -rd.requestUrl.split("/").count(_.nonEmpty)) diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700Assets.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700Assets.scala new file mode 100644 index 0000000000..818bd46f18 --- /dev/null +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700Assets.scala @@ -0,0 +1,203 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.v7_0_0 + +import cats.effect.IO +import code.api.Constant.ApiPathZero +import code.api.util.APIUtil.{EmptyBody, ResourceDoc, unboxFullOrFail} +import code.api.util.ApiTag._ +import code.api.util.ErrorMessages._ +import code.api.util.CustomJsonFormats +import code.api.util.http4s.Http4sRequestAttributes.EndpointHelpers +import code.asset.{AssetStatuses, AssetTypes, Assets} +import code.util.Helper +import com.github.dwickern.macros.NameOf.nameOf +import com.openbankproject.commons.ExecutionContext.Implicits.global +import com.openbankproject.commons.util.ApiVersion +import org.http4s._ +import org.http4s.dsl.io._ +import org.json4s.Formats + +import scala.collection.mutable.ArrayBuffer +import scala.concurrent.Future +import scala.util.Try + +/** + * This object holds the v7.0.0 endpoints that read the asset registry: the currencies, precious + * metals, accounting units, crypto assets and bank-issued assets that amounts can be held in (see + * [[code.asset.Assets]] and ideas/ASSET_REGISTRY.md). + * + * They need no authentication: the registry is reference data, and an app may need to check a code + * before anyone has logged in. Nothing here writes. + * + * It is declared in its own object to keep Http4s700's initialiser under the JVM's 64KB method limit. + */ +object Http4s700Assets { + + implicit val formats: Formats = CustomJsonFormats.formats + + private val implementedInApiVersion = ApiVersion.v7_0_0 + private val prefixPath = Root / ApiPathZero.toString / implementedInApiVersion.toString + + val resourceDocs = ArrayBuffer[ResourceDoc]() + + /** The most assets one page returns, and the default page size: large enough for every seeded asset. */ + val MaxPageSize = 500 + + // Route: GET /obp/v7.0.0/assets + lazy val getAssets: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "assets" => + EndpointHelpers.executeAndRespond(req) { cc => + val parameters = req.uri.query.params + def filter(name: String): Option[String] = parameters.get(name).map(_.trim).filter(_.nonEmpty) + val assetType = filter("asset_type").map(_.toUpperCase) + val status = filter("status").map(_.toUpperCase) + val chainScheme = filter("chain_scheme").map(_.toUpperCase) + val issuerBankId = filter("issuer_bank_id") + val limit = filter("limit").map(value => Try(value.toInt).getOrElse(-1)).getOrElse(MaxPageSize) + val offset = filter("offset").map(value => Try(value.toInt).getOrElse(-1)).getOrElse(0) + for { + _ <- Helper.booleanToFuture(s"$InvalidAssetQueryParameter asset_type must be one of ${AssetTypes.all.toList.sorted.mkString(", ")}.", cc = Some(cc)) { + assetType.forall(AssetTypes.all.contains) + } + _ <- Helper.booleanToFuture(s"$InvalidAssetQueryParameter status must be one of ${AssetStatuses.all.toList.sorted.mkString(", ")}.", cc = Some(cc)) { + status.forall(AssetStatuses.all.contains) + } + _ <- Helper.booleanToFuture(s"$InvalidAssetQueryParameter limit must be a whole number from 1 to $MaxPageSize.", cc = Some(cc)) { + limit >= 1 && limit <= MaxPageSize + } + _ <- Helper.booleanToFuture(s"$InvalidAssetQueryParameter offset must be a whole number, 0 or more.", cc = Some(cc)) { + offset >= 0 + } + (assets, total) <- Future(Assets.getAssets(assetType, issuerBankId, status, chainScheme, limit, offset)) + } yield JSONFactory700Assets.createAssetsJson(assets, total, limit, offset) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getAssets), + "GET", + "/assets", + "Get Assets", + s"""Lists the assets in this instance's asset registry: the units an amount can be held in. That is + |every currency, precious metal and accounting unit in ISO 4217, the crypto assets XBT, ADA and ETH, + |and assets issued by a bank, such as deposit tokens, stablecoins, bonds or fund shares. + | + |For each asset the response gives its code (the value that appears in `currency` fields), its type, + |how many decimal places its amounts have, the bank that issued it (none for currencies, metals, + |accounting units and crypto assets, which are administered at the `SYS` bank), and its status: + |`ACTIVE`, `SUSPENDED` or `RETIRED`. + | + |The registry does not yet decide which currency codes OBP accepts or how many decimal places it + |gives them; it shows what it will decide once it does. + | + |Query parameters, all optional; a filter matches exactly, apart from letter case for the first three: + | + |- `asset_type`: one of ${AssetTypes.all.toList.sorted.mkString(", ")} + |- `status`: one of ${AssetStatuses.all.toList.sorted.mkString(", ")}. Without it, assets of every status are listed. + |- `chain_scheme`: the chain and network an asset is recorded on, e.g. `CARDANO_MAINNET` + |- `issuer_bank_id`: the bank that issued the asset + |- `limit`: the most assets to return, from 1 to $MaxPageSize (default $MaxPageSize) + |- `offset`: how many assets to skip (default 0) + | + |Assets are ordered by code. `pagination.total` is the number of assets matching the filters. + | + |No Authentication is Required.""".stripMargin, + EmptyBody, + JSONFactory700Assets.assetsJsonExample, + List(InvalidAssetQueryParameter, UnknownError), + List(apiTagAsset), + None, + http4sPartialFunction = Some(getAssets) + ) + + // Route: GET /obp/v7.0.0/assets/ASSET_CODE + lazy val getAsset: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "assets" / assetCode if assetCode.nonEmpty => + EndpointHelpers.executeAndRespond(req) { cc => + Future(Assets.getAsset(assetCode)) + .map(unboxFullOrFail(_, Some(cc), s"$AssetNotFound Current value is $assetCode", 404)) + .map(JSONFactory700Assets.createAssetJson) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getAsset), + "GET", + "/assets/ASSET_CODE", + "Get Asset", + s"""Returns one asset from the asset registry by its code, for example `EUR`, `XAU` or `ETH`. + |The code is matched ignoring letter case: `eur` returns `EUR`. + | + |See Get Assets for what the fields mean. + | + |No Authentication is Required.""".stripMargin, + EmptyBody, + JSONFactory700Assets.assetJsonExample, + List(AssetNotFound, UnknownError), + List(apiTagAsset), + None, + http4sPartialFunction = Some(getAsset) + ) + + // Route: GET /obp/v7.0.0/assets/chain/CHAIN_SCHEME/CHAIN_ASSET_ID + lazy val getAssetByChainIdentity: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "assets" / "chain" / chainScheme / chainAssetId if chainScheme.nonEmpty && chainAssetId.nonEmpty => + EndpointHelpers.executeAndRespond(req) { cc => + Future(Assets.getAssetByChainIdentity(chainScheme.toUpperCase, chainAssetId)) + .map(unboxFullOrFail(_, Some(cc), s"$AssetNotFoundByChainIdentity Current value is $chainScheme/$chainAssetId", 404)) + .map(JSONFactory700Assets.createAssetJson) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getAssetByChainIdentity), + "GET", + "/assets/chain/CHAIN_SCHEME/CHAIN_ASSET_ID", + "Get Asset by Chain Identity", + s"""Returns the asset recorded on a blockchain under this identity. It answers the question "a + |transfer of this token arrived; which asset is it?", for reconciling OBP balances with the chain. + | + |`CHAIN_SCHEME` names both the chain and the network, because the same identity can be a different + |token on another network: for example `CARDANO_MAINNET`, `CARDANO_PREPROD` or `ETHEREUM_SEPOLIA`. + |It is matched ignoring letter case. `CHAIN_ASSET_ID` is the token's identity on that network: on + |Cardano, the policy id and the hex asset name joined by a dot; on Ethereum, the token's contract address. + | + |Only tokens have a chain identity. A chain's own currency, such as ADA or ETH, has none. + | + |No Authentication is Required.""".stripMargin, + EmptyBody, + JSONFactory700Assets.issuedAssetJsonExample, + List(AssetNotFoundByChainIdentity, UnknownError), + List(apiTagAsset), + None, + http4sPartialFunction = Some(getAssetByChainIdentity) + ) +} diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory700Assets.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory700Assets.scala new file mode 100644 index 0000000000..c4bed7d434 --- /dev/null +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory700Assets.scala @@ -0,0 +1,121 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.v7_0_0 + +import java.util.Date + +import code.asset.AssetTrait + +/* + * The JSON of the v7.0.0 asset registry endpoints. Package-level case classes, for the reason given in + * JSONFactory700Operations. + */ + +/** One asset in the registry: a unit amounts can be held in, such as EUR, XAU, ETH or a bank's deposit token. */ +case class AssetJsonV700( + asset_id: String, + asset_code: String, + asset_type: String, + name: String, + decimal_places: Int, + issuer_bank_id: Option[String], + issuer_product_code: Option[String], + chain_scheme: Option[String], + chain_asset_id: Option[String], + dti: Option[String], + status: String, + created_by_user_id: String, + created_at: Date, + updated_at: Date +) + +case class AssetsPaginationJsonV700(total: Int, limit: Int, offset: Int) + +case class AssetsJsonV700(assets: List[AssetJsonV700], pagination: AssetsPaginationJsonV700) + +object JSONFactory700Assets { + + def createAssetJson(asset: AssetTrait): AssetJsonV700 = + AssetJsonV700( + asset_id = asset.assetId, + asset_code = asset.assetCode, + asset_type = asset.assetType, + name = asset.name, + decimal_places = asset.decimalPlaces, + issuer_bank_id = asset.issuerBankId, + issuer_product_code = asset.issuerProductCode, + chain_scheme = asset.chainScheme, + chain_asset_id = asset.chainAssetId, + dti = asset.dti, + status = asset.status, + created_by_user_id = asset.createdByUserId, + created_at = asset.createdAt, + updated_at = asset.updatedAt + ) + + def createAssetsJson(assets: List[AssetTrait], total: Int, limit: Int, offset: Int): AssetsJsonV700 = + AssetsJsonV700(assets.map(createAssetJson), AssetsPaginationJsonV700(total, limit, offset)) + + /** A seeded currency, as the registry holds it today. */ + val assetJsonExample: AssetJsonV700 = AssetJsonV700( + asset_id = "7a1c3e52-9b0d-4f6a-8c2e-1d5b9f0a3c47", + asset_code = "EUR", + asset_type = "FIAT", + name = "Euro", + decimal_places = 2, + issuer_bank_id = None, + issuer_product_code = None, + chain_scheme = None, + chain_asset_id = None, + dti = None, + status = "ACTIVE", + created_by_user_id = "system:asset-seed", + created_at = new Date(), + updated_at = new Date() + ) + + /** A token issued by a bank and recorded on Cardano, the shape the reverse chain lookup returns. */ + val issuedAssetJsonExample: AssetJsonV700 = AssetJsonV700( + asset_id = "0c9e4b7d-2f61-4a8e-b3d5-6a1f8e2c9b04", + asset_code = "TZBOND29", + asset_type = "DEBT_SECURITY", + name = "Example Bank 2029 Fixed Rate Note", + decimal_places = 0, + issuer_bank_id = Some("example.bank.tz"), + issuer_product_code = Some("TZBOND29"), + chain_scheme = Some("CARDANO_MAINNET"), + chain_asset_id = Some("f0ff48bbb7bbe9d59a40f1ce90e9e9d0ff5002ec48f232b49ca0fb9a.545a424f4e443239"), + dti = None, + status = "ACTIVE", + created_by_user_id = "9ca9a7e4-6d02-40e3-a129-0b2bf89de9b1", + created_at = new Date(), + updated_at = new Date() + ) + + val assetsJsonExample: AssetsJsonV700 = + AssetsJsonV700(List(assetJsonExample), AssetsPaginationJsonV700(total = 1, limit = 500, offset = 0)) +} diff --git a/obp-api/src/main/scala/code/asset/Asset.scala b/obp-api/src/main/scala/code/asset/Asset.scala new file mode 100644 index 0000000000..d10f66fbf9 --- /dev/null +++ b/obp-api/src/main/scala/code/asset/Asset.scala @@ -0,0 +1,117 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.asset + +import code.util.MappedUUID +import net.liftweb.mapper._ + +/** + * This class is one row of the asset registry: a unit that amounts can be held in, such as a + * currency (EUR), a precious metal (XAU), a crypto asset (ETH) or an asset a bank issues (a deposit + * token, a bond). Its `AssetCode` is the value that appears in `currency` fields. + * + * The design is in ideas/ASSET_REGISTRY.md. The registry is seeded at boot (see [[AssetSeed]]), and + * `APIUtil.isValidCurrencyISOCode` and `Helper.currencyDecimalPlaces` read it through [[AssetLookup]]. + * + * An empty string in an optional column means "not set"; the accessors on [[AssetTrait]] turn it + * into `None`. + */ +class Asset extends AssetTrait with LongKeyedMapper[Asset] with IdPK { + def getSingleton = Asset + + object AssetId extends MappedUUID(this) + /** Always stored upper case, so a lookup that upper-cases its input ignores case. */ + object AssetCode extends MappedString(this, 10) + object AssetType extends MappedString(this, 32) + object Name extends MappedString(this, 125) + object DecimalPlaces extends MappedInt(this) + object IssuerBankId extends MappedString(this, 255) + object IssuerProductCode extends MappedString(this, 50) + object ChainScheme extends MappedString(this, 64) + object ChainAssetId extends MappedString(this, 255) + object Dti extends MappedString(this, 9) + object Status extends MappedString(this, 16) + object CreatedByUserId extends MappedString(this, 255) + object CreationDate extends MappedDateTime(this) { + override def defaultValue = new java.util.Date() + } + object LastUpdate extends MappedDateTime(this) { + override def defaultValue = new java.util.Date() + } + + override def assetId: String = AssetId.get + override def assetCode: String = AssetCode.get + override def assetType: String = AssetType.get + override def name: String = Name.get + override def decimalPlaces: Int = DecimalPlaces.get + override def issuerBankId: Option[String] = nonEmpty(IssuerBankId.get) + override def issuerProductCode: Option[String] = nonEmpty(IssuerProductCode.get) + override def chainScheme: Option[String] = nonEmpty(ChainScheme.get) + override def chainAssetId: Option[String] = nonEmpty(ChainAssetId.get) + override def dti: Option[String] = nonEmpty(Dti.get) + override def status: String = Status.get + override def createdByUserId: String = CreatedByUserId.get + override def createdAt: java.util.Date = CreationDate.get + override def updatedAt: java.util.Date = LastUpdate.get + + private def nonEmpty(value: String): Option[String] = Option(value).filter(_.nonEmpty) +} + +/** + * The unique indexes on (chain_scheme, chain_asset_id) and on dti that the design describes are not + * declared here: they must ignore rows where the value is not set, and Mapper cannot declare such a + * partial index. Nothing writes those columns yet; the write endpoints will enforce the uniqueness. + */ +object Asset extends Asset with LongKeyedMetaMapper[Asset] { + override def dbTableName = "Asset" + override def dbIndexes = UniqueIndex(AssetCode) :: UniqueIndex(AssetId) :: Index(IssuerBankId) :: super.dbIndexes +} + +/** + * This class is one change of an asset's status, for example ACTIVE to SUSPENDED, with who made it, + * when and why. The design keeps this separate from the metrics because metrics can be switched off + * and record API calls rather than state changes. Nothing writes it yet: the seed creates assets as + * ACTIVE, which is not a change, and there is no endpoint that changes a status. + */ +class AssetStatusHistory extends LongKeyedMapper[AssetStatusHistory] with IdPK { + def getSingleton = AssetStatusHistory + + object AssetId extends MappedString(this, 36) + object FromStatus extends MappedString(this, 16) + object ToStatus extends MappedString(this, 16) + object Reason extends MappedText(this) + object ChangedByUserId extends MappedString(this, 255) + object ChangedAt extends MappedDateTime(this) { + override def defaultValue = new java.util.Date() + } +} + +object AssetStatusHistory extends AssetStatusHistory with LongKeyedMetaMapper[AssetStatusHistory] { + override def dbTableName = "AssetStatusHistory" + override def dbIndexes = Index(AssetId) :: super.dbIndexes +} diff --git a/obp-api/src/main/scala/code/asset/AssetLookup.scala b/obp-api/src/main/scala/code/asset/AssetLookup.scala new file mode 100644 index 0000000000..b8a113872c --- /dev/null +++ b/obp-api/src/main/scala/code/asset/AssetLookup.scala @@ -0,0 +1,92 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.asset + +import code.api.util.APIUtil +import code.util.Helper +import code.util.Helper.MdcLoggable +import net.liftweb.common.Full +import net.liftweb.util.Helpers.tryo + +/** + * This object answers the two questions OBP asks about a currency code on almost every request + * that carries an amount: is the code known, and how many decimal places does it have. It answers + * them from the asset registry ([[Assets]]), so that `APIUtil.isValidCurrencyISOCode` and + * `Helper.currencyDecimalPlaces` no longer depend on a list built into the code. + * + * The answers are the same as before the registry existed (ideas/ASSET_REGISTRY.md, progress step 6): + * + * - Codes are matched exactly as written, so `EUR` is known and `eur` is not. Accepting any letter + * case is a later, deliberate change (section 4). + * - The status of an asset is not checked yet: a suspended or retired code is still known. That + * also comes later, when `isValidCurrencyISOCode` becomes `isUsableAsset` (section 4). + * - Three spellings the built-in list accepts are not registry codes: `ada` (the registry holds + * `ADA`), and `lovelace` and `wei`, the smallest units of ADA and ETH. They stay known, with the + * built-in decimal places, until amounts in those units are converted (section 7, Part C). + * - A code the registry does not hold gets the built-in decimal places, as it did before. + * + * The registry is read once into memory and read again after any write through [[Assets]]. If it + * cannot be read (no database, as in pure unit tests) or is still empty (before the boot seed has + * run), the built-in answers are used and nothing is kept, so the next call tries the registry again. + * Until the precisions are corrected (section 7, Part B) the built-in answers and the registry's + * agree for every code, because the seed copies them. + */ +object AssetLookup extends MdcLoggable { + + /** The spellings the built-in list accepts that are not codes in the registry. */ + val LegacySpellings: Set[String] = Set("ada", "lovelace", "wei") + + /** The decimal places of every registered asset, keyed by its code exactly as stored. */ + @volatile private var decimalPlacesByCode: Option[Map[String, Int]] = None + + /** Forgets what was read, so the next lookup reads the registry again. Called after every write. */ + def invalidate(): Unit = decimalPlacesByCode = None + + private def registry: Option[Map[String, Int]] = decimalPlacesByCode.orElse { + tryo(Assets.getAssets().map(asset => asset.assetCode -> asset.decimalPlaces).toMap) match { + case Full(loaded) if loaded.nonEmpty => + decimalPlacesByCode = Some(loaded) + decimalPlacesByCode + case Full(_) => + logger.debug("registry says: the asset registry is empty; using the built-in currency list") + None + case failure => + logger.debug(s"registry says: could not read the asset registry, using the built-in currency list: $failure") + None + } + } + + /** Whether OBP knows this currency code, matched exactly as written. */ + def isKnownCode(code: String): Boolean = registry match { + case Some(codes) => codes.contains(code) || LegacySpellings.contains(code) + case None => APIUtil.builtInCurrencyCodes.contains(code) + } + + /** The number of decimal places of this currency code, matched exactly as written. */ + def decimalPlaces(code: String): Int = + registry.flatMap(_.get(code)).getOrElse(Helper.builtInCurrencyDecimalPlaces(code)) +} diff --git a/obp-api/src/main/scala/code/asset/AssetSeed.scala b/obp-api/src/main/scala/code/asset/AssetSeed.scala new file mode 100644 index 0000000000..ea8c62e271 --- /dev/null +++ b/obp-api/src/main/scala/code/asset/AssetSeed.scala @@ -0,0 +1,113 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.asset + +import code.api.util.APIUtil +import code.util.Helper +import code.util.Helper.MdcLoggable +import net.liftweb.common.Full + +/** + * This object fills the asset registry with the assets every OBP instance knows: the currencies, + * precious metals and accounting units in media/xml/ISOCurrencyCodes.xml, plus the crypto assets + * XBT, ADA and ETH. It runs at every boot (Boot.scala) and only inserts codes that are missing; it + * never changes an existing row, so a deployment's own changes (a suspended code, a corrected + * name) survive a restart. Several instances booting at once are safe: the unique index on the code + * lets only one insert win, and the others count the code as already present. + * + * Every seeded asset gets the number of decimal places OBP uses for its code today + * (`Helper.builtInCurrencyDecimalPlaces`), not the ISO value, so that switching the lookups over to the + * registry (AssetLookup) changes nothing. Correcting precisions is a separate, later step + * (ideas/ASSET_REGISTRY.md, section 7, Part B). + * + * There is deliberately no property to switch the seed off: a deployment that does not handle a + * code suspends it in the registry, so there is one model and one audit trail. + */ +object AssetSeed extends MdcLoggable { + + /** The value stored in `created_by_user_id` for seeded rows; there is no real User behind them. */ + val SeedActor = "system:asset-seed" + + case class Entry(assetCode: String, assetType: String, name: String, decimalPlaces: Int) + + private val PreciousMetalCodes = Set("XAU", "XAG", "XPT", "XPD") + + /** + * The crypto entries in the XML file. They are not taken from the file: `ada` is seeded as `ADA` + * with the crypto assets below, and `lovelace` and `wei` are the smallest units of ADA and ETH, + * not assets of their own, so they are not seeded at all. + */ + private val CryptoCodesInXmlFile = Set("ADA", "LOVELACE", "ETH", "WEI") + + val cryptoEntries: List[Entry] = List( + Entry("XBT", AssetTypes.CRYPTO, "Bitcoin", Helper.builtInCurrencyDecimalPlaces("XBT")), + Entry("ADA", AssetTypes.CRYPTO, "Cardano ADA", Helper.builtInCurrencyDecimalPlaces("ADA")), + Entry("ETH", AssetTypes.CRYPTO, "Ether", Helper.builtInCurrencyDecimalPlaces("ETH")) + ) + + /** + * One entry per distinct code in the XML file, other than the crypto codes. A code that appears + * for several countries (EUR, USD) takes the currency name from its first entry. An ISO minor unit + * of "N.A." marks an accounting unit (XDR, XTS, ...), except for the four precious metals. + */ + lazy val isoEntries: List[Entry] = { + val rows = (APIUtil.CurrencyIsoCodeFromXmlFile \ "CcyTbl" \ "CcyNtry").toList + .map(row => ((row \ "Ccy").text.trim, (row \ "CcyNm").text.trim, (row \ "CcyMnrUnts").text.trim)) + .filter { case (code, _, _) => code.nonEmpty && !CryptoCodesInXmlFile.contains(code.toUpperCase) } + rows.groupBy(_._1).toList.map { case (code, rowsForCode) => + val (_, name, minorUnits) = rowsForCode.head + val assetType = + if (PreciousMetalCodes.contains(code)) AssetTypes.PRECIOUS_METAL + else if (minorUnits == "N.A.") AssetTypes.ACCOUNTING_UNIT + else AssetTypes.FIAT + Entry(code, assetType, name, Helper.builtInCurrencyDecimalPlaces(code)) + }.sortBy(_.assetCode) + } + + def entries: List[Entry] = isoEntries ++ cryptoEntries + + /** Inserts every missing entry and returns how many were inserted, already present, and failed. */ + def run(): (Int, Int, Int) = { + val existingCodes = Assets.getAssets().map(_.assetCode).toSet + val (inserted, alreadyPresent, failed) = entries.foldLeft((0, 0, 0)) { + case ((inserted, alreadyPresent, failed), entry) => + if (existingCodes.contains(entry.assetCode)) (inserted, alreadyPresent + 1, failed) + else Assets.createAsset(entry.assetCode, entry.assetType, entry.name, entry.decimalPlaces, + issuerBankId = None, status = AssetStatuses.ACTIVE, createdByUserId = SeedActor) match { + case Full(_) => (inserted + 1, alreadyPresent, failed) + // Another instance booting at the same time may have inserted it first. + case _ if Assets.getAsset(entry.assetCode).isDefined => (inserted, alreadyPresent + 1, failed) + case failure => + logger.warn(s"run says: could not seed asset ${entry.assetCode}: $failure") + (inserted, alreadyPresent, failed + 1) + } + } + logger.info(s"run says: inserted=$inserted alreadyPresent=$alreadyPresent failed=$failed (of ${entries.size} seed entries)") + (inserted, alreadyPresent, failed) + } +} diff --git a/obp-api/src/main/scala/code/asset/Assets.scala b/obp-api/src/main/scala/code/asset/Assets.scala new file mode 100644 index 0000000000..76cf399e64 --- /dev/null +++ b/obp-api/src/main/scala/code/asset/Assets.scala @@ -0,0 +1,200 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.asset + +import net.liftweb.common.{Box, Failure, Full} +import net.liftweb.mapper.{Ascending, By, MaxRows, OrderBy, QueryParam, StartAt} +import net.liftweb.util.Helpers.tryo + +/** This trait is what the rest of OBP sees of one asset in the registry; see [[Asset]]. */ +trait AssetTrait { + def assetId: String + def assetCode: String + def assetType: String + def name: String + def decimalPlaces: Int + def issuerBankId: Option[String] + def issuerProductCode: Option[String] + def chainScheme: Option[String] + def chainAssetId: Option[String] + def dti: Option[String] + def status: String + def createdByUserId: String + def createdAt: java.util.Date + def updatedAt: java.util.Date +} + +/** + * This object lists the kinds of asset the registry knows. Fiat currencies, precious metals, + * accounting units and crypto assets have no issuer and are administered at the SYS bank; every + * other kind is issued by a bank, which administers it. + */ +object AssetTypes { + val FIAT = "FIAT" + val PRECIOUS_METAL = "PRECIOUS_METAL" + val ACCOUNTING_UNIT = "ACCOUNTING_UNIT" + val CRYPTO = "CRYPTO" + val DEPOSIT_TOKEN = "DEPOSIT_TOKEN" + val STABLECOIN = "STABLECOIN" + val DEBT_SECURITY = "DEBT_SECURITY" + val EQUITY = "EQUITY" + val FUND_SHARE = "FUND_SHARE" + val OTHER = "OTHER" + + val withoutIssuer: Set[String] = Set(FIAT, PRECIOUS_METAL, ACCOUNTING_UNIT, CRYPTO) + val issued: Set[String] = Set(DEPOSIT_TOKEN, STABLECOIN, DEBT_SECURITY, EQUITY, FUND_SHARE, OTHER) + val all: Set[String] = withoutIssuer ++ issued +} + +/** This object lists the statuses an asset can have. Only ACTIVE assets may be used for new accounts and transactions. */ +object AssetStatuses { + val ACTIVE = "ACTIVE" + val SUSPENDED = "SUSPENDED" + val RETIRED = "RETIRED" + + val all: Set[String] = Set(ACTIVE, SUSPENDED, RETIRED) +} + +/** This trait is the set of operations on the asset registry. */ +trait AssetProvider { + /** Finds an asset by its code, ignoring case: `eur` finds `EUR`. */ + def getAsset(assetCode: String): Box[AssetTrait] + def getAssets(): List[AssetTrait] + /** + * Returns one page of the assets matching every filter given, ordered by code, and how many assets + * match in total. A filter of `None` matches everything. + */ + def getAssets( + assetType: Option[String], + issuerBankId: Option[String], + status: Option[String], + chainScheme: Option[String], + limit: Int, + offset: Int + ): (List[AssetTrait], Int) + /** Finds the asset with this on-chain identity, for example a Cardano token's policy id and asset name. */ + def getAssetByChainIdentity(chainScheme: String, chainAssetId: String): Box[AssetTrait] + def createAsset( + assetCode: String, + assetType: String, + name: String, + decimalPlaces: Int, + issuerBankId: Option[String], + status: String, + createdByUserId: String + ): Box[AssetTrait] +} + +/** + * This object is the asset registry stored in the OBP database. It checks the rules every asset + * must follow before it writes a row, so a bad row cannot be created by the seed or, later, by an + * endpoint. + */ +object Assets extends AssetProvider { + + /** Codes are 3 to 10 upper case letters or digits once upper-cased. */ + private val AssetCodePattern = "^[A-Z0-9]{3,10}$".r + + /** + * Amounts will be stored as DECIMAL(38, 18) (ideas/ASSET_REGISTRY.md, section 5), so no asset can + * use more than 18 decimal places. + */ + val MaxDecimalPlaces = 18 + + def normaliseCode(assetCode: String): String = assetCode.trim.toUpperCase + + override def getAsset(assetCode: String): Box[AssetTrait] = + Asset.find(By(Asset.AssetCode, normaliseCode(assetCode))) + + override def getAssets(): List[AssetTrait] = + Asset.findAll(OrderBy(Asset.AssetCode, Ascending)) + + override def getAssets( + assetType: Option[String], + issuerBankId: Option[String], + status: Option[String], + chainScheme: Option[String], + limit: Int, + offset: Int + ): (List[AssetTrait], Int) = { + val filters: List[QueryParam[Asset]] = + assetType.map(value => By(Asset.AssetType, value)).toList ::: + issuerBankId.map(value => By(Asset.IssuerBankId, value)).toList ::: + status.map(value => By(Asset.Status, value)).toList ::: + chainScheme.map(value => By(Asset.ChainScheme, value)).toList + val total = Asset.count(filters: _*).toInt + val page = Asset.findAll((filters :+ OrderBy(Asset.AssetCode, Ascending) :+ StartAt[Asset](offset) :+ MaxRows[Asset](limit)): _*) + (page, total) + } + + override def getAssetByChainIdentity(chainScheme: String, chainAssetId: String): Box[AssetTrait] = + if (chainScheme.isEmpty || chainAssetId.isEmpty) Failure("A chain identity needs both a chain scheme and a chain asset id.") + else Asset.find(By(Asset.ChainScheme, chainScheme), By(Asset.ChainAssetId, chainAssetId)) + + override def createAsset( + assetCode: String, + assetType: String, + name: String, + decimalPlaces: Int, + issuerBankId: Option[String], + status: String, + createdByUserId: String + ): Box[AssetTrait] = { + val code = normaliseCode(assetCode) + val issuer = issuerBankId.map(_.trim).filter(_.nonEmpty) + if (AssetCodePattern.findFirstIn(code).isEmpty) + Failure(s"Asset code '$assetCode' must be 3 to 10 letters or digits.") + else if (!AssetTypes.all.contains(assetType)) + Failure(s"Asset type '$assetType' must be one of ${AssetTypes.all.toList.sorted.mkString(", ")}.") + else if (decimalPlaces < 0 || decimalPlaces > MaxDecimalPlaces) + Failure(s"Decimal places $decimalPlaces must be between 0 and $MaxDecimalPlaces.") + else if (AssetTypes.issued.contains(assetType) && issuer.isEmpty) + Failure(s"An asset of type $assetType must have an issuer bank.") + else if (AssetTypes.withoutIssuer.contains(assetType) && issuer.nonEmpty) + Failure(s"An asset of type $assetType has no issuer bank; it is administered at the SYS bank.") + else if (!AssetStatuses.all.contains(status)) + Failure(s"Status '$status' must be one of ${AssetStatuses.all.toList.sorted.mkString(", ")}.") + else + tryo { + Asset.create + .AssetCode(code) + .AssetType(assetType) + .Name(name) + .DecimalPlaces(decimalPlaces) + .IssuerBankId(issuer.getOrElse("")) + .Status(status) + .CreatedByUserId(createdByUserId) + .saveMe() + } match { + case created @ Full(_) => + AssetLookup.invalidate() + created + case other => other + } + } +} diff --git a/obp-api/src/main/scala/code/util/Helper.scala b/obp-api/src/main/scala/code/util/Helper.scala index 8cfb716a76..9d149685ce 100644 --- a/obp-api/src/main/scala/code/util/Helper.scala +++ b/obp-api/src/main/scala/code/util/Helper.scala @@ -151,11 +151,16 @@ object Helper extends Loggable { /** * Returns the number of decimal places a currency has. E.g. "EUR" -> 2, "JPY" -> 0 - * - * @param currencyCode - * @return + * The answer comes from the asset registry; see code.asset.AssetLookup. + */ + def currencyDecimalPlaces(currencyCode : String): Int = code.asset.AssetLookup.decimalPlaces(currencyCode) + + /** + * This is the list of decimal places OBP used before the asset registry existed. The registry is + * seeded from it (code.asset.AssetSeed), and AssetLookup falls back to it when the registry cannot + * be read or does not hold a code. Use currencyDecimalPlaces everywhere else. */ - def currencyDecimalPlaces(currencyCode : String) = { + def builtInCurrencyDecimalPlaces(currencyCode : String): Int = { //this data was sourced from Wikipedia, so it might not all be correct, //and some banking systems may still retain different units (e.g. CZK?) //notable it doesn't cover non-traditional currencies (e.g. cryptocurrencies) diff --git a/obp-api/src/test/scala/code/api/v7_0_0/AssetsEndpointTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/AssetsEndpointTest.scala new file mode 100644 index 0000000000..f6eaaffc72 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v7_0_0/AssetsEndpointTest.scala @@ -0,0 +1,180 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.v7_0_0 + +import code.api.util.ErrorMessages.{AssetNotFound, AssetNotFoundByChainIdentity, InvalidAssetQueryParameter} +import code.api.v6_0_0.V600ServerSetup +import code.asset.{Asset, AssetSeed, AssetStatuses, AssetTypes, Assets, RestoresSeededAssetRegistry} +import com.openbankproject.commons.model.ErrorMessage +import com.openbankproject.commons.util.ApiVersion +import org.json4s.JsonAST.{JInt, JString, JValue} +import net.liftweb.mapper.By +import org.scalatest.Tag + +/** + * This suite checks the three read-only asset registry endpoints: they answer without + * authentication, list and filter the seeded registry, find one asset by code ignoring case, find a + * token by its chain identity, and refuse a bad query parameter with 400 rather than ignore it. + * + * The test setup empties every table after each test, so each scenario seeds the registry itself, + * and the suite puts the seeded registry back when it ends (RestoresSeededAssetRegistry). + */ +class AssetsEndpointTest extends V600ServerSetup with RestoresSeededAssetRegistry { + + def v7_0_0_Request = baseRequest / "obp" / "v7.0.0" + object VersionOfApi extends Tag(ApiVersion.v7_0_0.toString) + object ApiEndpoint extends Tag("assets") + + private def assets = v7_0_0_Request / "assets" + + private def seedRegistry(): Unit = { + Asset.bulkDelete_!!() + AssetSeed.run() + } + + private def codesIn(body: JValue): List[String] = + (body \ "assets").children.map(asset => (asset \ "asset_code").asInstanceOf[JString].s) + + private def total(body: JValue): BigInt = + (body \ "pagination" \ "total").asInstanceOf[JInt].num + + /** Registers a token issued by bank-a and records it on Cardano mainnet, as the write endpoints will later. */ + private val tokenChainAssetId = "f0ff48bbb7bbe9d59a40f1ce90e9e9d0ff5002ec48f232b49ca0fb9a.545a424f4e443239" + private def registerCardanoToken(): Unit = { + Assets.createAsset("TZBOND29", AssetTypes.DEBT_SECURITY, "Example 2029 Note", 0, Some("bank-a"), AssetStatuses.ACTIVE, "user-1") + .openOrThrowException("the token is created") + Asset.find(By(Asset.AssetCode, "TZBOND29")).openOrThrowException("the token exists") + .ChainScheme("CARDANO_MAINNET").ChainAssetId(tokenChainAssetId).saveMe() + } + + feature(s"Get Assets - GET /obp/v7.0.0/assets - $VersionOfApi") { + + scenario("anyone can list the seeded registry, ordered by code", ApiEndpoint, VersionOfApi) { + seedRegistry() + val response = makeGetRequest(assets.GET) + response.code should equal(200) + val codes = codesIn(response.body) + total(response.body) should equal(AssetSeed.entries.size) + codes.size should equal(AssetSeed.entries.size) + codes should equal(codes.sorted) + val eur = (response.body \ "assets").children.find(asset => (asset \ "asset_code") == JString("EUR")).get + (eur \ "asset_type") should equal(JString("FIAT")) + (eur \ "decimal_places").values should equal(2) + (eur \ "status") should equal(JString("ACTIVE")) + } + + scenario("filters match ignoring the letter case of the value", ApiEndpoint, VersionOfApi) { + seedRegistry() + val metals = makeGetRequest(assets.GET < "precious_metal")) + metals.code should equal(200) + codesIn(metals.body) should equal(List("XAG", "XAU", "XPD", "XPT")) + + val suspended = makeGetRequest(assets.GET < "SUSPENDED")) + suspended.code should equal(200) + total(suspended.body) should equal(0) + } + + scenario("issuer_bank_id and chain_scheme find a bank's token", ApiEndpoint, VersionOfApi) { + seedRegistry() + registerCardanoToken() + codesIn(makeGetRequest(assets.GET < "bank-a")).body) should equal(List("TZBOND29")) + codesIn(makeGetRequest(assets.GET < "cardano_mainnet")).body) should equal(List("TZBOND29")) + codesIn(makeGetRequest(assets.GET < "bank-b")).body) should equal(Nil) + } + + scenario("limit and offset page through the list, and total counts every match", ApiEndpoint, VersionOfApi) { + seedRegistry() + val allCodes = codesIn(makeGetRequest(assets.GET).body) + val page = makeGetRequest(assets.GET < "2", "offset" -> "1")) + page.code should equal(200) + codesIn(page.body) should equal(allCodes.slice(1, 3)) + total(page.body) should equal(AssetSeed.entries.size) + } + + scenario("a bad query parameter is refused with 400, not ignored", ApiEndpoint, VersionOfApi) { + seedRegistry() + val badParameters = List( + "asset_type" -> "BOGUS", + "status" -> "FROZEN", + "limit" -> "0", + "limit" -> "501", + "limit" -> "ten", + "offset" -> "-1" + ) + badParameters.foreach { parameter => + val response = makeGetRequest(assets.GET < + val response = makeGetRequest((assets / code).GET) + withClue(code) { + response.code should equal(404) + response.body.extract[ErrorMessage].message should startWith(AssetNotFound) + } + } + } + } + + feature(s"Get Asset by Chain Identity - GET /obp/v7.0.0/assets/chain/CHAIN_SCHEME/CHAIN_ASSET_ID - $VersionOfApi") { + + scenario("a registered token is found by its chain and network and its identity there", ApiEndpoint, VersionOfApi) { + seedRegistry() + registerCardanoToken() + val response = makeGetRequest((assets / "chain" / "cardano_mainnet" / tokenChainAssetId).GET) + response.code should equal(200) + (response.body \ "asset_code") should equal(JString("TZBOND29")) + (response.body \ "issuer_bank_id") should equal(JString("bank-a")) + (response.body \ "chain_scheme") should equal(JString("CARDANO_MAINNET")) + } + + scenario("the same identity on another network is not found", ApiEndpoint, VersionOfApi) { + seedRegistry() + registerCardanoToken() + val response = makeGetRequest((assets / "chain" / "CARDANO_PREPROD" / tokenChainAssetId).GET) + response.code should equal(404) + response.body.extract[ErrorMessage].message should startWith(AssetNotFoundByChainIdentity) + } + } +} diff --git a/obp-api/src/test/scala/code/asset/AssetLookupTest.scala b/obp-api/src/test/scala/code/asset/AssetLookupTest.scala new file mode 100644 index 0000000000..3e9b3a0664 --- /dev/null +++ b/obp-api/src/test/scala/code/asset/AssetLookupTest.scala @@ -0,0 +1,78 @@ +package code.asset + +import code.api.util.APIUtil +import code.setup.ServerSetup +import code.util.Helper + +/** + * This class tests that `APIUtil.isValidCurrencyISOCode` and `Helper.currencyDecimalPlaces` answer + * from the asset registry, through AssetLookup, and that their answers are the ones OBP gave before + * the registry existed (ideas/ASSET_REGISTRY.md, progress step 6). + * + * CurrencyHandlingTest checks the same functions without a database, where AssetLookup uses the + * built-in list; this suite checks them with the registry seeded. + * + * Each scenario seeds the registry itself and tells AssetLookup to forget what it read before, and + * the suite puts the seeded registry back when it ends (RestoresSeededAssetRegistry). + */ +class AssetLookupTest extends ServerSetup with RestoresSeededAssetRegistry { + + private def seedRegistry(): Unit = { + Asset.bulkDelete_!!() + AssetSeed.run() + AssetLookup.invalidate() + } + + /** Every code the built-in list knows, plus values it rejects or treats differently by letter case. */ + private val codesToCompare: List[String] = + APIUtil.builtInCurrencyCodes.toList.sorted ++ + List("eur", "Eur", "jpy", "kwd", "xbt", "eth", "LOVELACE", "WEI", "", "EUR USD", "978", "USDC", "MRO") + + feature("Answers from the seeded registry") { + + scenario("Every code is accepted or rejected as the built-in list does, except that ADA is now accepted") { + seedRegistry() + val differences = codesToCompare.filter(code => APIUtil.isValidCurrencyISOCode(code) != APIUtil.builtInCurrencyCodes.contains(code)) + differences shouldBe Nil + Then("ADA, the registry's code for Cardano's currency, is accepted, and so is the built-in spelling ada") + APIUtil.isValidCurrencyISOCode("ADA") shouldBe true + APIUtil.isValidCurrencyISOCode("ada") shouldBe true + } + + scenario("Every code has the decimal places of the built-in table") { + seedRegistry() + val differences = (codesToCompare :+ "ADA").collect { + case code if Helper.currencyDecimalPlaces(code) != Helper.builtInCurrencyDecimalPlaces(code) => + s"$code: registry ${Helper.currencyDecimalPlaces(code)}, built-in ${Helper.builtInCurrencyDecimalPlaces(code)}" + } + differences shouldBe Nil + } + } + + feature("The registry is what is read") { + + scenario("An asset added to the registry is accepted, with the registry's decimal places") { + seedRegistry() + APIUtil.isValidCurrencyISOCode("TOKENAB") shouldBe false + + When("a bank registers a deposit token with 4 decimal places") + Assets.createAsset("TOKENAB", AssetTypes.DEPOSIT_TOKEN, "Token AB", 4, Some("bank-a"), AssetStatuses.ACTIVE, "user-1") + .openOrThrowException("the token is created") + + Then("the code is accepted at once, with 4 decimal places") + APIUtil.isValidCurrencyISOCode("TOKENAB") shouldBe true + Helper.currencyDecimalPlaces("TOKENAB") shouldBe 4 + Helper.convertToSmallestCurrencyUnits(BigDecimal("1.2345"), "TOKENAB") shouldBe 12345L + } + + scenario("With the registry empty, the built-in answers are used") { + Asset.bulkDelete_!!() + AssetLookup.invalidate() + APIUtil.isValidCurrencyISOCode("EUR") shouldBe true + APIUtil.isValidCurrencyISOCode("lovelace") shouldBe true + APIUtil.isValidCurrencyISOCode("ADA") shouldBe false + Helper.currencyDecimalPlaces("JPY") shouldBe 0 + Helper.currencyDecimalPlaces("KWD") shouldBe 3 + } + } +} diff --git a/obp-api/src/test/scala/code/asset/AssetSeedTest.scala b/obp-api/src/test/scala/code/asset/AssetSeedTest.scala new file mode 100644 index 0000000000..bad81d70f9 --- /dev/null +++ b/obp-api/src/test/scala/code/asset/AssetSeedTest.scala @@ -0,0 +1,140 @@ +package code.asset + +import code.setup.ServerSetup +import code.util.Helper +import net.liftweb.mapper.By + +/** + * This class tests the asset registry's seed and the rules the registry enforces on every row. + * + * The seed's job at this stage is to reproduce exactly what OBP answered before the registry: every + * code in the built-in list (`APIUtil.builtInCurrencyCodes`) is seeded (apart from `lovelace` and + * `wei`, which are units of ADA and ETH, and `ada`, which is seeded upper case), each at the decimal + * places of the built-in table (`Helper.builtInCurrencyDecimalPlaces`). These scenarios, with + * AssetLookupTest, are what show that reading the registry instead changed nothing. + * + * Each scenario empties the asset table and runs the seed itself rather than relying on the seed + * run at boot, and the suite puts the seeded registry back when it ends (RestoresSeededAssetRegistry). + */ +class AssetSeedTest extends ServerSetup with RestoresSeededAssetRegistry { + + private def emptyRegistryAndSeed(): (Int, Int, Int) = { + Asset.bulkDelete_!!() + AssetSeed.run() + } + + feature("The seed entries") { + scenario("Every code in the currency XML file is seeded, except the crypto units, plus XBT, ADA and ETH") { + val seededCodes = AssetSeed.entries.map(_.assetCode) + seededCodes.distinct.size shouldBe seededCodes.size + // 183 codes in the file, minus ada, lovelace, ETH and wei, plus XBT, ADA and ETH. + seededCodes.size shouldBe 182 + seededCodes should contain allOf ("EUR", "USD", "JPY", "KWD", "XAU", "XDR", "XBT", "ADA", "ETH") + seededCodes should contain noneOf ("ada", "lovelace", "LOVELACE", "wei", "WEI") + } + + scenario("Every entry has the decimal places OBP uses for that code today") { + AssetSeed.entries.foreach { entry => + withClue(entry.assetCode) { entry.decimalPlaces shouldBe Helper.builtInCurrencyDecimalPlaces(entry.assetCode) } + } + } + + scenario("Each code gets the right asset type") { + val typeByCode = AssetSeed.entries.map(entry => entry.assetCode -> entry.assetType).toMap + typeByCode("EUR") shouldBe AssetTypes.FIAT + typeByCode("XAU") shouldBe AssetTypes.PRECIOUS_METAL + typeByCode("XAG") shouldBe AssetTypes.PRECIOUS_METAL + typeByCode("XDR") shouldBe AssetTypes.ACCOUNTING_UNIT + typeByCode("XTS") shouldBe AssetTypes.ACCOUNTING_UNIT + typeByCode("XBT") shouldBe AssetTypes.CRYPTO + typeByCode("ADA") shouldBe AssetTypes.CRYPTO + typeByCode("ETH") shouldBe AssetTypes.CRYPTO + } + } + + feature("Seeding the registry") { + scenario("A first run inserts every entry as an ACTIVE asset with no issuer") { + val (inserted, alreadyPresent, failed) = emptyRegistryAndSeed() + inserted shouldBe AssetSeed.entries.size + alreadyPresent shouldBe 0 + failed shouldBe 0 + + val assetsByCode = Assets.getAssets().map(asset => asset.assetCode -> asset).toMap + assetsByCode.keySet shouldBe AssetSeed.entries.map(_.assetCode).toSet + AssetSeed.entries.foreach { entry => + val asset = assetsByCode(entry.assetCode) + withClue(entry.assetCode) { + asset.assetType shouldBe entry.assetType + asset.name shouldBe entry.name + asset.decimalPlaces shouldBe entry.decimalPlaces + asset.status shouldBe AssetStatuses.ACTIVE + asset.issuerBankId shouldBe None + asset.createdByUserId shouldBe AssetSeed.SeedActor + asset.assetId should not be empty + } + } + } + + scenario("A second run inserts nothing") { + emptyRegistryAndSeed() + val (inserted, alreadyPresent, failed) = AssetSeed.run() + inserted shouldBe 0 + alreadyPresent shouldBe AssetSeed.entries.size + failed shouldBe 0 + Asset.count shouldBe AssetSeed.entries.size + } + + scenario("A run never changes an asset that is already there") { + emptyRegistryAndSeed() + Asset.find(By(Asset.AssetCode, "XTS")).openOrThrowException("XTS was seeded") + .Status(AssetStatuses.SUSPENDED).Name("Changed by this deployment").saveMe() + + AssetSeed.run() + + val xts = Assets.getAsset("XTS").openOrThrowException("XTS is still there") + xts.status shouldBe AssetStatuses.SUSPENDED + xts.name shouldBe "Changed by this deployment" + } + } + + feature("Looking up and creating assets") { + scenario("A lookup ignores the case of the code") { + emptyRegistryAndSeed() + Assets.getAsset("eur").map(_.assetCode) shouldBe Assets.getAsset("EUR").map(_.assetCode) + Assets.getAsset("Ada").map(_.assetCode).toOption shouldBe Some("ADA") + Assets.getAsset("lovelace").isDefined shouldBe false + } + + scenario("A code is stored upper case") { + Asset.bulkDelete_!!() + Assets.createAsset("tokenab", AssetTypes.DEPOSIT_TOKEN, "Token AB", 2, Some("bank-a"), AssetStatuses.ACTIVE, "user-1") + .map(_.assetCode).toOption shouldBe Some("TOKENAB") + } + + scenario("Assets that break the registry's rules are refused") { + Asset.bulkDelete_!!() + def create(code: String, assetType: String, decimalPlaces: Int, issuer: Option[String], status: String = AssetStatuses.ACTIVE) = + Assets.createAsset(code, assetType, "Name", decimalPlaces, issuer, status, "user-1") + + create("AB", AssetTypes.CRYPTO, 2, None).isDefined shouldBe false // too short + create("ABCDEFGHIJK", AssetTypes.CRYPTO, 2, None).isDefined shouldBe false // too long + create("AB-C", AssetTypes.CRYPTO, 2, None).isDefined shouldBe false // not a letter or digit + create("ABCD", "NOT_A_TYPE", 2, None).isDefined shouldBe false + create("ABCD", AssetTypes.CRYPTO, -1, None).isDefined shouldBe false + create("ABCD", AssetTypes.CRYPTO, 19, None).isDefined shouldBe false // beyond the storage scale + create("ABCD", AssetTypes.DEPOSIT_TOKEN, 2, None).isDefined shouldBe false // issued type needs an issuer + create("ABCD", AssetTypes.FIAT, 2, Some("bank-a")).isDefined shouldBe false // fiat has no issuer + create("ABCD", AssetTypes.CRYPTO, 2, None, status = "FROZEN").isDefined shouldBe false + Asset.count shouldBe 0 + + create("ABCD", AssetTypes.CRYPTO, 18, None).isDefined shouldBe true + } + + scenario("A code can only be registered once, whatever its case") { + Asset.bulkDelete_!!() + Assets.createAsset("ABCD", AssetTypes.CRYPTO, "First", 2, None, AssetStatuses.ACTIVE, "user-1").isDefined shouldBe true + Assets.createAsset("abcd", AssetTypes.CRYPTO, "Second", 2, None, AssetStatuses.ACTIVE, "user-1").isDefined shouldBe false + Asset.count shouldBe 1 + } + } +} diff --git a/obp-api/src/test/scala/code/asset/RestoresSeededAssetRegistry.scala b/obp-api/src/test/scala/code/asset/RestoresSeededAssetRegistry.scala new file mode 100644 index 0000000000..67ec1a05ba --- /dev/null +++ b/obp-api/src/test/scala/code/asset/RestoresSeededAssetRegistry.scala @@ -0,0 +1,20 @@ +package code.asset + +import org.scalatest.{BeforeAndAfterAll, Suite} + +/** + * This trait is for suites that empty the asset registry or fill it with test assets. Currency + * validation and decimal places read the registry (AssetLookup), and a suite that runs later in the + * same JVM without resetting the database, such as the pure unit test CurrencyHandlingTest, would + * otherwise read whatever the last scenario left there. When the suite ends, the registry is put + * back to what the boot seed makes, and AssetLookup forgets its in-memory copy. + */ +trait RestoresSeededAssetRegistry extends BeforeAndAfterAll { this: Suite => + override def afterAll(): Unit = { + try { + Asset.bulkDelete_!!() + AssetSeed.run() + AssetLookup.invalidate() + } finally super.afterAll() + } +} diff --git a/obp-api/src/test/scala/code/setup/ServerSetup.scala b/obp-api/src/test/scala/code/setup/ServerSetup.scala index e09b0c704a..46252dc6ef 100644 --- a/obp-api/src/test/scala/code/setup/ServerSetup.scala +++ b/obp-api/src/test/scala/code/setup/ServerSetup.scala @@ -148,6 +148,8 @@ trait ServerSetup extends FeatureSpec with SendServerRequests logger.warn(s"[TEST ISOLATION] Failed to clear table for ${model.getClass.getSimpleName}: ${e.getMessage}") } } + // The reset empties the asset registry behind AssetLookup's in-memory copy, so forget the copy too. + code.asset.AssetLookup.invalidate() } val server = TestServer @@ -225,6 +227,9 @@ trait ServerSetupWithTestData extends ServerSetup with DefaultConnectorTestSetup override def afterEach() = { super.afterEach() wipeTestData() + // The wipe empties the asset registry behind AssetLookup's in-memory copy. Forget the copy, or a + // registry a scenario left holding only a test asset would decide currency codes for later suites. + code.asset.AssetLookup.invalidate() } } \ No newline at end of file diff --git a/scripts/asset_registry_currency_report.sql b/scripts/asset_registry_currency_report.sql new file mode 100644 index 0000000000..a4b49cb2b6 --- /dev/null +++ b/scripts/asset_registry_currency_report.sql @@ -0,0 +1,114 @@ +-- This script reports how stored amounts are distributed across currency codes, so the asset +-- registry migration (ideas/ASSET_REGISTRY.md, section 7) can be sized against real data before +-- any of it is written. It only reads: everything runs in one transaction that is rolled back at +-- the end, and that transaction is switched to read-only before the first query reads data, so +-- even a mistake in this file cannot change data. The only things it creates are two temporary +-- views, which exist only in this session and are discarded by the rollback. +-- +-- Run it against a copy of the database you want to assess (PostgreSQL), for example: +-- psql -h localhost -U obp -d sandbox -f scripts/asset_registry_currency_report.sql +-- +-- Background: amounts are stored as whole numbers of minor units, at the precision +-- Helper.currencyDecimalPlaces gives each code today (CZK/JPY/KRW 0, KWD/OMR 3, every other code 2). +-- The migration converts them to exact decimals in the main unit, then corrects each code's +-- precision. Lowering a code's precision is only possible if no stored amount in that code has +-- digits beyond the new precision, and that is what sections 3 and 4 count. + +BEGIN; + +-- Every stored amount, with the currency it is in and the table and column it came from. +-- bankaccountbalance has no currency of its own; it takes the currency of its account. +CREATE TEMPORARY VIEW stored_amounts AS + SELECT 'mappedbankaccount.accountbalance'::text AS amount_column, accountcurrency AS currency, accountbalance AS minor_units FROM mappedbankaccount +UNION ALL SELECT 'mappedtransaction.amount', currency, amount FROM mappedtransaction +UNION ALL SELECT 'mappedtransaction.newaccountbalance', currency, newaccountbalance FROM mappedtransaction +UNION ALL SELECT 'standingorder.amountvalue', amountcurrency, amountvalue FROM standingorder +UNION ALL SELECT 'bankaccountbalance.balanceamount', account.accountcurrency, balance.balanceamount + FROM bankaccountbalance balance + LEFT JOIN mappedbankaccount account + ON account.theaccountid = balance.accountid_ AND account.bank = balance.bankid_; + +-- The precision each code is stored at today (Helper.currencyDecimalPlaces). The match there is +-- case-sensitive, so 'jpy' is stored at 2, not 0; this view reproduces that exactly. +CREATE TEMPORARY VIEW stored_amounts_with_precision AS +SELECT amount_column, currency, minor_units, + CASE WHEN currency IN ('CZK', 'JPY', 'KRW') THEN 0 + WHEN currency IN ('KWD', 'OMR') THEN 3 + ELSE 2 END AS legacy_decimal_places +FROM stored_amounts; + +-- PostgreSQL refuses CREATE (even of a temporary view) in a read-only transaction, so the views +-- above are created first and the transaction is made read-only here, before anything is read. +SET LOCAL transaction_read_only = on; + +\echo +\echo '1. Rows per currency code and amount column (every code with stored amounts)' +SELECT currency, amount_column, count(*) AS row_count +FROM stored_amounts +GROUP BY currency, amount_column +ORDER BY currency, amount_column; + +\echo +\echo '2. Currency values that are not in upper case, or that are missing' +\echo ' (affected by the rule that currency codes are case-insensitive and stored upper case)' +SELECT coalesce(currency, '') AS currency, amount_column, count(*) AS row_count +FROM stored_amounts +WHERE currency IS NULL OR currency <> upper(currency) +GROUP BY currency, amount_column +ORDER BY currency, amount_column; + +\echo +\echo '3. Codes whose ISO precision is 0 but which are stored at 2:' +\echo ' rows whose amount has a fractional part, which would block lowering the precision to 0' +SELECT currency, amount_column, + count(*) AS row_count, + count(*) FILTER (WHERE minor_units % 100 <> 0) AS rows_with_fractional_amount +FROM stored_amounts +WHERE currency IN ('BIF', 'CLP', 'DJF', 'GNF', 'ISK', 'KMF', 'PYG', 'RWF', 'UGX', 'UYI', + 'VND', 'VUV', 'XAF', 'XOF', 'XPF') +GROUP BY currency, amount_column +ORDER BY currency, amount_column; + +\echo +\echo '4. lovelace and wei (to be folded into ADA and ETH): rows holding a fractional lovelace or' +\echo ' wei, which is not a real on-chain amount and would be reported, not converted' +SELECT currency, amount_column, + count(*) AS row_count, + count(*) FILTER (WHERE minor_units % 100 <> 0) AS rows_with_fractional_unit +FROM stored_amounts +WHERE lower(currency) IN ('lovelace', 'wei') +GROUP BY currency, amount_column +ORDER BY currency, amount_column; + +\echo +\echo '5. Conversion check: the largest absolute amount per code, in minor units and in the main unit' +\echo ' (the new column holds up to 20 digits before the decimal point, so every Long fits)' +SELECT currency, legacy_decimal_places, + max(abs(minor_units)) AS largest_minor_units, + max(abs(minor_units))::numeric / (10 ^ legacy_decimal_places)::numeric AS largest_main_units +FROM stored_amounts_with_precision +WHERE minor_units IS NOT NULL +GROUP BY currency, legacy_decimal_places +ORDER BY currency; + +\echo +\echo '6. Balance records whose account cannot be found (their currency is unknown, so they cannot be converted)' +SELECT count(*) AS orphaned_balance_rows +FROM bankaccountbalance balance +LEFT JOIN mappedbankaccount account + ON account.theaccountid = balance.accountid_ AND account.bank = balance.bankid_ +WHERE account.id IS NULL; + +\echo +\echo '7. Amounts already stored as decimals or whole units, outside the five minor-unit columns' +\echo ' productfee.amount is a decimal with 2 places whatever the currency;' +\echo ' mappedtransactiontype.mcustomerfee_amount is a whole number of main units' +SELECT 'productfee.amount' AS amount_column, currency, count(*) AS row_count, + count(*) FILTER (WHERE amount <> trunc(amount)) AS rows_with_fractional_amount +FROM productfee GROUP BY currency +UNION ALL +SELECT 'mappedtransactiontype.mcustomerfee_amount', mcustomerfee_currency, count(*), 0 +FROM mappedtransactiontype GROUP BY mcustomerfee_currency +ORDER BY 1, 2; + +ROLLBACK; From 9c64de2a3290011a97f3d8e41c8dcf24a6f5e4e7 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 5 Oct 2026 23:13:14 +0200 Subject: [PATCH 14/16] fix: keep a bare obp_exists[X] / obp_not_exists[X] key as a join A key with no '=' arrived with no values and was dropped. It is now a join with no predicate. Test: JoinQuerySpec. + feature Files. --- .../main/scala/bootstrap/liftweb/Boot.scala | 11 + obp-api/src/main/scala/code/files/Files.scala | 123 +++++++++ .../code/files/MappedFilesProvider.scala | 238 ++++++++++++++++++ .../dynamic/entity/query/JoinQuerySpec.scala | 11 + 4 files changed, 383 insertions(+) create mode 100644 obp-api/src/main/scala/code/files/Files.scala create mode 100644 obp-api/src/main/scala/code/files/MappedFilesProvider.scala diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index a68570e7c7..aad033ee35 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -327,6 +327,10 @@ class Boot extends MdcLoggable { // Toggle off via routing_schemes.seed_defaults_at_boot=false in environments that don't want defaults. code.routingscheme.RoutingSchemeSeed.runIfEnabled() + // Idempotent seed of the asset registry (ISO currencies, precious metals, accounting units, + // XBT, ADA, ETH) at the decimal places OBP uses today. Currency validation and decimal places read it. + code.asset.AssetSeed.run() + // Report which static Glossary Items the database is currently displacing. A developer editing // Glossary.scala has no other way to find out that their text is being overridden. code.api.util.Glossary.logStaticOverrides() @@ -1053,6 +1057,11 @@ object ToSchemify extends MdcLoggable { DynamicData, DynamicDataAccess, code.api.dynamic.entity.projection.DynamicEntityIndex, + // Files: written in full because Boot imports java.io.File. + code.files.File, + code.files.FileContent, + code.files.FileAttachment, + code.files.FileAccess, DynamicEndpoint, AccountIdMapping, DirectDebit, @@ -1141,6 +1150,8 @@ object ToSchemify extends MdcLoggable { Organisation, RoutingScheme, BankSupportedRoutingScheme, + code.asset.Asset, + code.asset.AssetStatusHistory, code.glossaryitem.DynamicGlossaryItem, code.platformapp.PlatformApp, code.platformapp.PlatformAppRequiredScope, diff --git a/obp-api/src/main/scala/code/files/Files.scala b/obp-api/src/main/scala/code/files/Files.scala new file mode 100644 index 0000000000..c5d519e12e --- /dev/null +++ b/obp-api/src/main/scala/code/files/Files.scala @@ -0,0 +1,123 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.files + +import java.util.Date + +import net.liftweb.common.Box +import net.liftweb.util.SimpleInjector + +/** + * This file holds the model for Files: documents such as PDFs and photographs that belong to a space + * (a bank, or SYS for the system space) and can be attached to OBP records. See the Glossary item + * "Files" and ideas/ogcr_file_storage.md. + * + * A file's bytes never change once stored. A new version of a document is a new file. What can + * change is which records the file is attached to and which Users have been given access to it. + * + * The metadata (File, FileAttachment, FileAccess) is kept apart from the bytes, which are reached + * only through a [[FileStore]]. Today that store is a Postgres table; it can be replaced by object + * storage without changing anything above it. + */ +object Files extends SimpleInjector { + val provider = new Inject(() => buildOne) {} + def buildOne: FilesProvider = MappedFilesProvider +} + +/** One stored file: what it is and who owns it. Never the bytes, which only the [[FileStore]] holds. */ +trait FileT { + def fileId: String + def bankId: String + /** Hex SHA-256 of the bytes, computed by OBP when the file was uploaded. */ + def sha256: String + def sizeInBytes: Long + def mediaType: String + def fileName: String + /** The User who uploaded the file: an agent's own id when an agent made the call. */ + def userId: String + /** The User an agent uploaded the file for. Empty when nobody was delegating. */ + def onBehalfOfUserId: String + def createdAt: Date + + /** The file's owner: the User an agent acted for, or else the User who uploaded it. */ + def ownerUserId: String = if (onBehalfOfUserId.nonEmpty) onBehalfOfUserId else userId +} + +/** A link between a file and one record, such as a Customer or a Dynamic Entity record. */ +trait FileAttachmentT { + def fileAttachmentId: String + def fileId: String + def bankId: String + def recordType: String + def recordId: String + def userId: String + def onBehalfOfUserId: String + def createdAt: Date +} + +/** A User's read access to one file, given by the file's owner. */ +trait FileAccessT { + def fileId: String + def granteeUserId: String + def grantedByUserId: String + def onBehalfOfUserId: String + def createdAt: Date +} + +/** + * This trait is where a file's bytes are kept. It is deliberately small, so that the Postgres store + * used today can be replaced by object storage (S3, MinIO, Azure Blob) later. Because the bytes never + * change and are identified by their hash, such a move is a copy followed by a hash check. + * + * `write` must take part in the caller's database transaction where the store is the database, so + * that a file's metadata and its bytes are saved together or not at all. + */ +trait FileStore { + def write(fileId: String, bytes: Array[Byte]): Box[Unit] + def read(fileId: String): Box[Array[Byte]] +} + +trait FilesProvider { + /** Store the bytes and the metadata of a new file, together. */ + def createFile(bankId: String, sha256: String, mediaType: String, fileName: String, + userId: String, onBehalfOfUserId: String, bytes: Array[Byte]): Box[FileT] + def getFile(bankId: String, fileId: String): Box[FileT] + def getContent(fileId: String): Box[Array[Byte]] + + def createAttachment(file: FileT, recordType: String, recordId: String, + userId: String, onBehalfOfUserId: String): Box[FileAttachmentT] + def getAttachment(bankId: String, fileId: String, fileAttachmentId: String): Box[FileAttachmentT] + def attachmentExists(fileId: String, recordType: String, recordId: String): Boolean + def deleteAttachment(fileAttachmentId: String): Box[Boolean] + /** The files attached to one record of a space. */ + def getFilesAttachedTo(bankId: String, recordType: String, recordId: String): List[FileT] + + def grantAccess(fileId: String, granteeUserId: String, grantedByUserId: String, onBehalfOfUserId: String): Box[FileAccessT] + def getAccess(fileId: String, granteeUserId: String): Box[FileAccessT] + def getAccessList(fileId: String): List[FileAccessT] + def revokeAccess(fileId: String, granteeUserId: String): Box[Boolean] +} diff --git a/obp-api/src/main/scala/code/files/MappedFilesProvider.scala b/obp-api/src/main/scala/code/files/MappedFilesProvider.scala new file mode 100644 index 0000000000..6227e88126 --- /dev/null +++ b/obp-api/src/main/scala/code/files/MappedFilesProvider.scala @@ -0,0 +1,238 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.files + +import java.util.{Date, UUID} + +import net.liftweb.common.{Box, Empty, Full} +import net.liftweb.mapper._ +import net.liftweb.util.Helpers.tryo + +/** + * This object stores Files with Lift Mapper: the metadata in File, FileAttachment and FileAccess, + * and the bytes, through [[PostgresFileStore]], in FileContent. + */ +object MappedFilesProvider extends FilesProvider { + + /** Where the bytes go. Replaced by an object storage implementation when the volume calls for it. */ + val store: FileStore = PostgresFileStore + + override def createFile(bankId: String, sha256: String, mediaType: String, fileName: String, + userId: String, onBehalfOfUserId: String, bytes: Array[Byte]): Box[FileT] = { + val fileId = UUID.randomUUID().toString + for { + _ <- store.write(fileId, bytes) + file <- tryo { + File.create + .FileId(fileId) + .BankId(bankId) + .Sha256(sha256) + .SizeInBytes(bytes.length.toLong) + .MediaType(mediaType) + .FileName(fileName) + .UserId(userId) + .OnBehalfOfUserId(onBehalfOfUserId) + .CreatedAt(new Date()) + .saveMe() + } + } yield file + } + + override def getFile(bankId: String, fileId: String): Box[FileT] = + File.find(By(File.BankId, bankId), By(File.FileId, fileId)) + + override def getContent(fileId: String): Box[Array[Byte]] = store.read(fileId) + + override def createAttachment(file: FileT, recordType: String, recordId: String, + userId: String, onBehalfOfUserId: String): Box[FileAttachmentT] = tryo { + FileAttachment.create + .FileId(file.fileId) + .BankId(file.bankId) + .RecordType(recordType) + .RecordId(recordId) + .UserId(userId) + .OnBehalfOfUserId(onBehalfOfUserId) + .CreatedAt(new Date()) + .saveMe() + } + + override def getAttachment(bankId: String, fileId: String, fileAttachmentId: String): Box[FileAttachmentT] = + FileAttachment.find(By(FileAttachment.BankId, bankId), By(FileAttachment.FileId, fileId), + By(FileAttachment.FileAttachmentId, fileAttachmentId)) + + override def attachmentExists(fileId: String, recordType: String, recordId: String): Boolean = + FileAttachment.find(By(FileAttachment.FileId, fileId), By(FileAttachment.RecordType, recordType), + By(FileAttachment.RecordId, recordId)).isDefined + + override def deleteAttachment(fileAttachmentId: String): Box[Boolean] = tryo { + FileAttachment.bulkDelete_!!(By(FileAttachment.FileAttachmentId, fileAttachmentId)) + } + + override def getFilesAttachedTo(bankId: String, recordType: String, recordId: String): List[FileT] = { + val fileIds = FileAttachment.findAll(By(FileAttachment.BankId, bankId), By(FileAttachment.RecordType, recordType), + By(FileAttachment.RecordId, recordId), OrderBy(FileAttachment.CreatedAt, Ascending)).map(_.FileId.get) + if (fileIds.isEmpty) Nil + else { + val byId = File.findAll(By(File.BankId, bankId), ByList(File.FileId, fileIds.distinct)).map(f => f.fileId -> f).toMap + fileIds.distinct.flatMap(byId.get) + } + } + + override def grantAccess(fileId: String, granteeUserId: String, grantedByUserId: String, onBehalfOfUserId: String): Box[FileAccessT] = tryo { + FileAccess.create + .FileId(fileId) + .GranteeUserId(granteeUserId) + .GrantedByUserId(grantedByUserId) + .OnBehalfOfUserId(onBehalfOfUserId) + .CreatedAt(new Date()) + .saveMe() + } + + override def getAccess(fileId: String, granteeUserId: String): Box[FileAccessT] = + FileAccess.find(By(FileAccess.FileId, fileId), By(FileAccess.GranteeUserId, granteeUserId)) + + override def getAccessList(fileId: String): List[FileAccessT] = + FileAccess.findAll(By(FileAccess.FileId, fileId), OrderBy(FileAccess.CreatedAt, Ascending)) + + override def revokeAccess(fileId: String, granteeUserId: String): Box[Boolean] = tryo { + FileAccess.bulkDelete_!!(By(FileAccess.FileId, fileId), By(FileAccess.GranteeUserId, granteeUserId)) + } +} + +/** + * This store keeps each file's bytes in the FileContent table. It suits files of a few megabytes + * at modest volume; the cost is that every database backup carries every file. The table is kept + * apart from File because Mapper reads every column of a row, so listings of File would otherwise + * load whole documents. + */ +object PostgresFileStore extends FileStore { + override def write(fileId: String, bytes: Array[Byte]): Box[Unit] = tryo { + FileContent.create.FileId(fileId).Content(bytes).saveMe() + () + } + + override def read(fileId: String): Box[Array[Byte]] = + FileContent.find(By(FileContent.FileId, fileId)).flatMap(row => Box !! row.Content.get) +} + +class File extends FileT with LongKeyedMapper[File] with IdPK { + override def getSingleton = File + + object FileId extends MappedString(this, 36) + object BankId extends MappedString(this, 255) + object Sha256 extends MappedString(this, 64) + object SizeInBytes extends MappedLong(this) + object MediaType extends MappedString(this, 100) + object FileName extends MappedString(this, 255) + object UserId extends MappedString(this, 255) + object OnBehalfOfUserId extends MappedString(this, 255) + object CreatedAt extends MappedDateTime(this) + + override def fileId: String = FileId.get + override def bankId: String = BankId.get + override def sha256: String = Sha256.get + override def sizeInBytes: Long = SizeInBytes.get + override def mediaType: String = MediaType.get + override def fileName: String = FileName.get + override def userId: String = UserId.get + override def onBehalfOfUserId: String = Option(OnBehalfOfUserId.get).getOrElse("") + override def createdAt: Date = CreatedAt.get +} + +object File extends File with LongKeyedMetaMapper[File] { + override def dbIndexes = UniqueIndex(FileId) :: Index(BankId, FileId) :: super.dbIndexes +} + +class FileContent extends LongKeyedMapper[FileContent] with IdPK { + override def getSingleton = FileContent + + object FileId extends MappedString(this, 36) + object Content extends MappedBinary(this) +} + +object FileContent extends FileContent with LongKeyedMetaMapper[FileContent] { + override def dbIndexes = UniqueIndex(FileId) :: super.dbIndexes +} + +class FileAttachment extends FileAttachmentT with LongKeyedMapper[FileAttachment] with IdPK { + override def getSingleton = FileAttachment + + object FileAttachmentId extends MappedString(this, 36) { + override def defaultValue = UUID.randomUUID().toString + } + object FileId extends MappedString(this, 36) + object BankId extends MappedString(this, 255) + object RecordType extends MappedString(this, 255) + object RecordId extends MappedString(this, 255) + object UserId extends MappedString(this, 255) + object OnBehalfOfUserId extends MappedString(this, 255) + object CreatedAt extends MappedDateTime(this) + + override def fileAttachmentId: String = FileAttachmentId.get + override def fileId: String = FileId.get + override def bankId: String = BankId.get + override def recordType: String = RecordType.get + override def recordId: String = RecordId.get + override def userId: String = UserId.get + override def onBehalfOfUserId: String = Option(OnBehalfOfUserId.get).getOrElse("") + override def createdAt: Date = CreatedAt.get +} + +object FileAttachment extends FileAttachment with LongKeyedMetaMapper[FileAttachment] { + override def dbIndexes = + UniqueIndex(FileAttachmentId) :: + // A file is attached to a record at most once. + UniqueIndex(FileId, RecordType, RecordId) :: + // "Which files belong to this record?" + Index(BankId, RecordType, RecordId) :: + super.dbIndexes +} + +class FileAccess extends FileAccessT with LongKeyedMapper[FileAccess] with IdPK { + override def getSingleton = FileAccess + + object FileId extends MappedString(this, 36) + object GranteeUserId extends MappedString(this, 255) + object GrantedByUserId extends MappedString(this, 255) + object OnBehalfOfUserId extends MappedString(this, 255) + object CreatedAt extends MappedDateTime(this) + + override def fileId: String = FileId.get + override def granteeUserId: String = GranteeUserId.get + override def grantedByUserId: String = GrantedByUserId.get + override def onBehalfOfUserId: String = Option(OnBehalfOfUserId.get).getOrElse("") + override def createdAt: Date = CreatedAt.get +} + +object FileAccess extends FileAccess with LongKeyedMetaMapper[FileAccess] { + override def dbIndexes = + // One row per User per file. + UniqueIndex(FileId, GranteeUserId) :: + // "Which files have been shared with me?" + Index(GranteeUserId) :: + super.dbIndexes +} diff --git a/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinQuerySpec.scala b/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinQuerySpec.scala index 53dfd927c1..62350e8e05 100644 --- a/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinQuerySpec.scala +++ b/obp-api/src/test/scala/code/api/dynamic/entity/query/JoinQuerySpec.scala @@ -84,6 +84,17 @@ class JoinQuerySpec extends FlatSpec with Matchers { joins shouldBe List(RawJoin(Quantifier.NotExists, "Contract", None, Nil)) } + it should "parse a bare join key with no '=' (no values) as a no-predicate join, not drop it" in { + // http4s multiParams gives `?obp_not_exists[Contract]` (no '=') an empty value list. + val query = org.http4s.Query.unsafeFromString("obp_not_exists[Contract]&obp_exists[Partner]") + val multi = query.multiParams.map { case (k, vs) => k -> vs.toList } + multi("obp_not_exists[Contract]") shouldBe Nil + val Right((_, joins, _, _)) = QueryParamParser.parse(multi) + joins should contain theSameElementsAs List( + RawJoin(Quantifier.NotExists, "Contract", None, Nil), + RawJoin(Quantifier.Exists, "Partner", None, Nil)) + } + it should "parse a nested predicate reusing the filter grammar" in { val Right((_, joins, _, _)) = QueryParamParser.parse(params("obp_exists[Contract]" -> "filter[active]=eq:true")) joins shouldBe List(RawJoin(Quantifier.Exists, "Contract", None, List(Filter("active", FilterOp.Eq, List("true"))))) From 2ed1614ce996f33becbef65417326697fa91679d Mon Sep 17 00:00:00 2001 From: simonredfern Date: Tue, 6 Oct 2026 06:13:08 +0200 Subject: [PATCH 15/16] fix: declare attribution for the asset and files user-id columns UserReferenceAttributionPolicyTest failed shard 8 on nine new columns. AssetStatusHistory.ChangedByUserId is audit; the rest record the human and are listed as not yet wired until their endpoints exist. --- .../src/main/scala/code/users/UserReference.scala | 12 ++++++++++++ .../api/sweep/OnBehalfOfOwnershipSweepTest.scala | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/obp-api/src/main/scala/code/users/UserReference.scala b/obp-api/src/main/scala/code/users/UserReference.scala index 76ad8e9613..85f7ba7e4b 100644 --- a/obp-api/src/main/scala/code/users/UserReference.scala +++ b/obp-api/src/main/scala/code/users/UserReference.scala @@ -161,6 +161,7 @@ object UserReference { case object UserRefreshes_UserId extends UserReference(UseAuthenticatedUserId, "code.UserRefreshes.MappedUserRefreshes", List("mUserId"), "operational: refresh of the authenticated user's own account list") case object PlatformApp_MarkedByUserId extends UserReference(UseAuthenticatedUserId, "code.platformapp.PlatformApp", List("MarkedByUserId"), "audit: who marked the Consumer as a platform app") case object GroupMembership_CreatedByUserId extends UserReference(UseAuthenticatedUserId, "code.group.GroupMembership", List("CreatedByUserId"), "audit: who added the member, as Entitlement_GrantedByUserId") + case object AssetStatusHistory_ChangedByUserId extends UserReference(UseAuthenticatedUserId, "code.asset.AssetStatusHistory", List("ChangedByUserId"), "audit: who changed the asset's status, the question a regulator asks") // ---- UseOnBehalfOfUserId: the row belongs to the person, so it must outlive the Consent that // ---- created it. A handful of these tables keep both ids, and those name two fields. @@ -201,6 +202,11 @@ object UserReference { case object DomainApi_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.domainapi.DomainApi", List("CreatedByUserId"), "the Domain API's creator; a published API outlives the Consent that created it") case object Bank_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.model.dataAccess.MappedBank", List("CreatedByUserId"), "creator grant already resolved at the endpoint") case object Organisation_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.organisation.Organisation", List("CreatedByUserId"), "outlives the Consent that created it") + case object Asset_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.asset.Asset", List("CreatedByUserId"), "outlives the Consent that created it, as RoutingScheme_CreatedByUserId; seeded rows hold system:asset-seed") + case object File_UserId extends UserReference(UseOnBehalfOfUserId , "code.files.File", List("UserId", "OnBehalfOfUserId"), "record both: UserId = who uploaded (the agent), OnBehalfOfUserId = the human, who owns the file") + case object FileAttachment_UserId extends UserReference(UseOnBehalfOfUserId , "code.files.FileAttachment", List("UserId", "OnBehalfOfUserId"), "record both: UserId = who attached (the agent), OnBehalfOfUserId = the human") + case object FileAccess_GrantedByUserId extends UserReference(UseOnBehalfOfUserId , "code.files.FileAccess", List("GrantedByUserId", "OnBehalfOfUserId"), "record both: GrantedByUserId = who granted (the agent), OnBehalfOfUserId = the file's owner it acted for") + case object FileAccess_GranteeUserId extends UserReference(UseOnBehalfOfUserId , "code.files.FileAccess", List("GranteeUserId"), "explicit target: the User given access is named in the request; access given to a consent user would die with its Consent") case object PayeeLookup_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.payeelookup.PayeeLookup", List("CreatedByUserId"), "outlives the Consent that created it") case object RoutingScheme_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.routingscheme.RoutingScheme", List("CreatedByUserId"), "outlives the Consent that created it") case object UtilityPaymentCallback_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.utilitypayment.UtilityPaymentCallback", List("CreatedByUserId"), "outlives the Consent that created it") @@ -255,6 +261,7 @@ object UserReference { UserRefreshes_UserId, PlatformApp_MarkedByUserId, GroupMembership_CreatedByUserId, + AssetStatusHistory_ChangedByUserId, TransactionRequest_UserId, Entitlement_UserId, GroupMembership_UserId, @@ -291,6 +298,11 @@ object UserReference { DomainApi_CreatedByUserId, Bank_CreatedByUserId, Organisation_CreatedByUserId, + Asset_CreatedByUserId, + File_UserId, + FileAttachment_UserId, + FileAccess_GrantedByUserId, + FileAccess_GranteeUserId, PayeeLookup_CreatedByUserId, RoutingScheme_CreatedByUserId, UtilityPaymentCallback_CreatedByUserId, diff --git a/obp-api/src/test/scala/code/api/sweep/OnBehalfOfOwnershipSweepTest.scala b/obp-api/src/test/scala/code/api/sweep/OnBehalfOfOwnershipSweepTest.scala index fedadf915a..a1ac159dd7 100644 --- a/obp-api/src/test/scala/code/api/sweep/OnBehalfOfOwnershipSweepTest.scala +++ b/obp-api/src/test/scala/code/api/sweep/OnBehalfOfOwnershipSweepTest.scala @@ -100,6 +100,10 @@ class OnBehalfOfOwnershipSweepTest extends ServerSetupWithTestData with DefaultU "deferred 2026-09-16, not mechanical: agreed as record-both (like Counterparty), but the webhook " + "code has two dead paths already and is not to be touched in a hurry. todo/webhook_attribution.md." + private val filesNotYetServed = + "the Files tables (code.files) exist but no endpoint writes them yet. The endpoints must store " + + "cc.onBehalfOfUserId as the owner (ideas/ogcr_file_storage.md) and reject a consent user as grantee." + private val mechanicalBatch = "Phase 2 mechanical batch: the provider does not call attributionOf yet, so a consent user's " + "row is stored against the consent user and dies with the Consent." @@ -141,6 +145,12 @@ class OnBehalfOfOwnershipSweepTest extends ServerSetupWithTestData with DefaultU "ApiProductSubscription_CreatedByUserId" -> mechanicalBatch, "DynamicGlossaryItem_CreatedByUserId" -> mechanicalBatch, "Organisation_CreatedByUserId" -> mechanicalBatch, + "Asset_CreatedByUserId" -> + "no endpoint writes the asset registry yet; only the boot seed does. Wire it with the asset write endpoints.", + "File_UserId" -> filesNotYetServed, + "FileAttachment_UserId" -> filesNotYetServed, + "FileAccess_GrantedByUserId" -> filesNotYetServed, + "FileAccess_GranteeUserId" -> filesNotYetServed, "PayeeLookup_CreatedByUserId" -> mechanicalBatch, "RoutingScheme_CreatedByUserId" -> mechanicalBatch, "UtilityPaymentCallback_CreatedByUserId" -> mechanicalBatch, From ac51d3be3f2d96bfe85753b783d6cd8a486f9341 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Tue, 6 Oct 2026 06:54:19 +0200 Subject: [PATCH 16/16] refactor: look up request headers only through RequestHeadersUtil, ignoring case. Improves the DAuth check in ApiSession and the limit/offset filter in Http4s510 that still compared exactly. HeaderLookupConventionsTest fails the build on a direct header name comparison; ConsentHeaderDispatchTest checks that the request path, not the letter case of Consent-Id / Consent-ID, picks OBP or Berlin Group consent rules. --- .../scala/code/api/DirectLoginRoutes.scala | 3 +- obp-api/src/main/scala/code/api/dauth.scala | 2 +- obp-api/src/main/scala/code/api/siwe.scala | 5 +- .../main/scala/code/api/util/APIUtil.scala | 50 ++++-------- .../main/scala/code/api/util/ApiSession.scala | 2 +- .../code/api/util/AuthorisationUtil.scala | 6 +- .../code/api/util/BerlinGroupCheck.scala | 16 ++-- .../code/api/util/BerlinGroupSigning.scala | 11 ++- .../scala/code/api/util/ConsentUtil.scala | 17 ++-- .../main/scala/code/api/util/JwsUtil.scala | 10 +-- .../code/api/util/RequestHeadersUtil.scala | 38 +++++++++ .../scala/code/api/util/WriteMetricUtil.scala | 2 +- .../scala/code/api/v5_1_0/Http4s510.scala | 2 +- .../scala/code/api/v6_0_0/Http4s600.scala | 3 +- .../api/util/ConsentHeaderDispatchTest.scala | 57 +++++++++++++ .../util/HeaderLookupConventionsTest.scala | 80 +++++++++++++++++++ 16 files changed, 227 insertions(+), 77 deletions(-) create mode 100644 obp-api/src/test/scala/code/api/util/ConsentHeaderDispatchTest.scala create mode 100644 obp-api/src/test/scala/code/api/util/HeaderLookupConventionsTest.scala diff --git a/obp-api/src/main/scala/code/api/DirectLoginRoutes.scala b/obp-api/src/main/scala/code/api/DirectLoginRoutes.scala index 35e8eb7c16..b269e44b8a 100644 --- a/obp-api/src/main/scala/code/api/DirectLoginRoutes.scala +++ b/obp-api/src/main/scala/code/api/DirectLoginRoutes.scala @@ -60,8 +60,7 @@ object DirectLoginRoutes { * instead of Lift's thread-local `S.request`. */ private def parseDirectLoginParams(cc: CallContext): Map[String, String] = { - def find(name: String): Option[String] = cc.requestHeaders - .find(_.name.equalsIgnoreCase(name)) + def find(name: String): Option[String] = code.api.util.RequestHeadersUtil.find(cc.requestHeaders, name) .flatMap(_.values.headOption) val directLoginHeader = find("DirectLogin") val authHeader = find("Authorization") diff --git a/obp-api/src/main/scala/code/api/dauth.scala b/obp-api/src/main/scala/code/api/dauth.scala index 60bc075c85..b1e0712a54 100755 --- a/obp-api/src/main/scala/code/api/dauth.scala +++ b/obp-api/src/main/scala/code/api/dauth.scala @@ -134,7 +134,7 @@ object DAuth extends MdcLoggable { // Check if the request (access token or request token) is valid and return a tuple def getDAuthToken(requestHeaders: List[HTTPParam]) : Option[List[String]] = { - requestHeaders.find(_.name.equalsIgnoreCase(APIUtil.DAuthHeaderKey)).map(_.values) + code.api.util.RequestHeadersUtil.find(requestHeaders, APIUtil.DAuthHeaderKey).map(_.values) } def getOrCreateResourceUser(jwtPayload: String, callContext: Option[CallContext]) : Box[(User, Option[CallContext])] = { diff --git a/obp-api/src/main/scala/code/api/siwe.scala b/obp-api/src/main/scala/code/api/siwe.scala index 95e980dca1..0b3130d5d8 100644 --- a/obp-api/src/main/scala/code/api/siwe.scala +++ b/obp-api/src/main/scala/code/api/siwe.scala @@ -127,12 +127,11 @@ object SIWE extends MdcLoggable { val SiweHeaderKey = "SIWE" def hasSiweHeader(requestHeaders: List[HTTPParam]): Boolean = - requestHeaders.exists(_.name.equalsIgnoreCase(SiweHeaderKey)) + code.api.util.RequestHeadersUtil.exists(requestHeaders, SiweHeaderKey) /** Parse `SIWE: token=` → Some(key). Mirrors DirectLogin's `token=` parsing. */ def getSiweToken(requestHeaders: List[HTTPParam]): Option[String] = { - val raw = requestHeaders - .find(_.name.equalsIgnoreCase(SiweHeaderKey)) + val raw = code.api.util.RequestHeadersUtil.find(requestHeaders, SiweHeaderKey) .flatMap(_.values.headOption) .getOrElse("") raw.split(",").map(_.trim).flatMap { entry => diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 7ab1316a35..178f9abe01 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -239,9 +239,9 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ def hasDirectLoginHeader(authorization: Box[String]): Boolean = hasHeader("DirectLogin", authorization) - def has2021DirectLoginHeader(requestHeaders: List[HTTPParam]): Boolean = requestHeaders.exists(_.name.equalsIgnoreCase("DirectLogin")) + def has2021DirectLoginHeader(requestHeaders: List[HTTPParam]): Boolean = RequestHeadersUtil.exists(requestHeaders, "DirectLogin") - def hasAuthorizationHeader(requestHeaders: List[HTTPParam]): Boolean = requestHeaders.exists(_.name.equalsIgnoreCase("Authorization")) + def hasAuthorizationHeader(requestHeaders: List[HTTPParam]): Boolean = RequestHeadersUtil.exists(requestHeaders, "Authorization") /* The OAuth 2.0 Authorization Framework: Bearer Token @@ -262,7 +262,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * Other types: the `GatewayLogin` is in the VALUE * Authorization:GatewayLogin token=xxxx */ - def hasDAuthHeader(requestHeaders: List[HTTPParam]) = requestHeaders.exists(_.name.equalsIgnoreCase(DAuthHeaderKey)) + def hasDAuthHeader(requestHeaders: List[HTTPParam]) = RequestHeadersUtil.exists(requestHeaders, DAuthHeaderKey) /** * Helper function which tells us does an "Authorization" request header field has the Type of an authentication scheme @@ -282,13 +282,9 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * @return the Consent-JWT value from a Request Header as a String */ def getConsentJWT(requestHeaders: List[HTTPParam]): Option[String] = { - requestHeaders.toSet.filter(_.name.equalsIgnoreCase(RequestHeader.`Consent-JWT`)).toList match { - case x :: Nil => Some(x.values.mkString(", ")) - case _ => requestHeaders.toSet.filter(_.name.equalsIgnoreCase(RequestHeader.`Consent-Id`)).toList match { - case x :: Nil => Some(x.values.mkString(", ")) - case _ => None - } - } + RequestHeadersUtil.findSingle(requestHeaders, RequestHeader.`Consent-JWT`) + .orElse(RequestHeadersUtil.findSingle(requestHeaders, RequestHeader.`Consent-Id`)) + .map(_.values.mkString(", ")) } /** @@ -296,27 +292,18 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * @return the Consent-JWT value from a Request Header as a String */ def getConsentIdRequestHeaderValue(requestHeaders: List[HTTPParam]): Option[String] = { - requestHeaders.toSet.filter(_.name.equalsIgnoreCase(RequestHeader.`Consent-Id`)).toList match { - case x :: Nil => Some(x.values.mkString(", ")) - case _ => None - } + RequestHeadersUtil.findSingle(requestHeaders, RequestHeader.`Consent-Id`).map(_.values.mkString(", ")) } /** * Purpose of this helper function is to get the PSD2-CERT value from a Request Headers. * @return the PSD2-CERT value from a Request Header as a String */ def `getPSD2-CERT`(requestHeaders: List[HTTPParam]): Option[String] = { - requestHeaders.toSet.filter(_.name.equalsIgnoreCase(RequestHeader.`PSD2-CERT`)).toList match { - case x :: Nil => Some(x.values.mkString(", ")) - case _ => None - } + RequestHeadersUtil.findSingle(requestHeaders, RequestHeader.`PSD2-CERT`).map(_.values.mkString(", ")) } def getRequestHeader(name: String, requestHeaders: List[HTTPParam]): String = { - requestHeaders.toSet.filter(_.name.equalsIgnoreCase(name)).toList match { - case x :: Nil => x.values.mkString(";") - case _ => "" - } + RequestHeadersUtil.findSingle(requestHeaders, name).map(_.values.mkString(";")).getOrElse("") } def hasConsentJWT(requestHeaders: List[HTTPParam]): Boolean = { @@ -329,10 +316,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * @return the Consent-ID value from a Request Header as a String */ def `getConsent-ID`(requestHeaders: List[HTTPParam]): Option[String] = { - requestHeaders.toSet.filter(_.name.equalsIgnoreCase(RequestHeader.`Consent-ID`)).toList match { - case x :: Nil => Some(x.values.mkString(", ")) - case _ => None - } + RequestHeadersUtil.findSingle(requestHeaders, RequestHeader.`Consent-ID`).map(_.values.mkString(", ")) } def `hasConsent-ID`(requestHeaders: List[HTTPParam]): Boolean = { `getConsent-ID`(requestHeaders).isDefined @@ -458,7 +442,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ val url = cc.map(_.url).getOrElse("") val requestHeaders: List[HTTPParam] = - cc.map(_.requestHeaders.filter(i => i.name == "limit" || i.name == "offset").sortBy(_.name)).getOrElse(Nil) + cc.map(_.requestHeaders.filter(i => RequestHeadersUtil.isNamed(i, "limit") || RequestHeadersUtil.isNamed(i, "offset")).sortBy(_.name)).getOrElse(Nil) val hashedRequestPayload = HashUtil.Sha256Hash(url + requestHeaders) val consumerId = cc.map(i => i.consumer.map(_.consumerId.get).getOrElse("None")).getOrElse("None") val userId = tryo(cc.map(i => i.userId).toBox).flatten.getOrElse("None") @@ -499,13 +483,13 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ private def checkConditionalRequest(cc: Option[CallContext], httpVerb: String, httpCode: Int, httpBody: Box[String]) = { val requestHeaders: List[HTTPParam] = cc.map(_.requestHeaders).getOrElse(Nil) - requestHeaders.filter(_.name.equalsIgnoreCase(RequestHeader.`If-None-Match`)).headOption match { + RequestHeadersUtil.find(requestHeaders, RequestHeader.`If-None-Match`) match { case Some(value) => // Handle the If-None-Match HTTP request header checkIfNotMatchHeader(cc, httpCode, httpBody, value.values.mkString("")) case None => // When used in combination with If-None-Match, it is ignored, unless the server doesn't support If-None-Match. // The most common use case is to update a cached entity that has no associated ETag - requestHeaders.filter(_.name.equalsIgnoreCase(RequestHeader.`If-Modified-Since`)).headOption match { + RequestHeadersUtil.find(requestHeaders, RequestHeader.`If-Modified-Since`) match { case Some(value) => // Handle the If-Modified-Since HTTP request header checkIfModifiedSinceHeader(cc, httpVerb, httpCode, httpBody, value.values.mkString("")) case None => @@ -2890,7 +2874,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ getRemoteIpAddress() val xRequestId: Option[String] = - reqHeaders.find(_.name.toLowerCase() == RequestHeader.`X-Request-ID`.toLowerCase()) + RequestHeadersUtil.find(reqHeaders, RequestHeader.`X-Request-ID`) .map(_.values.mkString(",")) logger.debug(s"Request Headers for verb: $verb, URL: $url") logger.debug(reqHeaders.map(h => h.name + ": " + h.values.mkString(",")).mkString) @@ -4791,7 +4775,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ * @return Full(errorResponse) if validate fail */ def validateRequestHeadersKeys(operationId: String, callContext: CallContext): Box[JsonResponse] = { - val headerKeysGrouped: Map[String, List[HTTPParam]] = callContext.requestHeaders.groupBy(_.name.toLowerCase(java.util.Locale.ROOT)) + val headerKeysGrouped: Map[String, List[HTTPParam]] = RequestHeadersUtil.groupByName(callContext.requestHeaders) headerKeysGrouped.toList.forall(_._2.size == 1) match { case true => Empty case false => @@ -4881,10 +4865,10 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ val requestHeaders = callContext.requestHeaders val forceError = requestHeaders.collectFirst { - case HTTPParam(name, value::_) if name.equalsIgnoreCase("Force-Error") => value + case header @ HTTPParam(_, value::_) if RequestHeadersUtil.isNamed(header, "Force-Error") => value } val responseCode = requestHeaders.collectFirst { - case HTTPParam(name, value::_) if name.equalsIgnoreCase("Response-Code") => value + case header @ HTTPParam(_, value::_) if RequestHeadersUtil.isNamed(header, "Response-Code") => value } if(forceError.isEmpty) { diff --git a/obp-api/src/main/scala/code/api/util/ApiSession.scala b/obp-api/src/main/scala/code/api/util/ApiSession.scala index 8f5894b297..63ff17004c 100644 --- a/obp-api/src/main/scala/code/api/util/ApiSession.scala +++ b/obp-api/src/main/scala/code/api/util/ApiSession.scala @@ -298,7 +298,7 @@ case class CallContext( def authType: AuthenticationType = { if(hasGatewayHeader(authReqHeaderField)) { GatewayLogin - } else if(requestHeaders.exists(_.name==DAuthHeaderKey)) { // DAuth Login + } else if(RequestHeadersUtil.exists(requestHeaders, DAuthHeaderKey)) { // DAuth Login DAuth } else if(has2021DirectLoginHeader(requestHeaders)) { // Direct Login DirectLogin diff --git a/obp-api/src/main/scala/code/api/util/AuthorisationUtil.scala b/obp-api/src/main/scala/code/api/util/AuthorisationUtil.scala index feb77fdeab..d72a61a513 100644 --- a/obp-api/src/main/scala/code/api/util/AuthorisationUtil.scala +++ b/obp-api/src/main/scala/code/api/util/AuthorisationUtil.scala @@ -32,9 +32,9 @@ import code.api.util.APIUtil.HTTPParam object AuthorisationUtil { def getAuthorisationHeaders(requestHeaders: List[HTTPParam]): List[String] = { - requestHeaders.map(_.name).filter(name => - List(`Consent-Id`, `Consent-ID`, `Consent-JWT`).exists(name.equalsIgnoreCase) - ) + requestHeaders + .filter(header => List(`Consent-Id`, `Consent-ID`, `Consent-JWT`).exists(RequestHeadersUtil.isNamed(header, _))) + .map(_.name) } diff --git a/obp-api/src/main/scala/code/api/util/BerlinGroupCheck.scala b/obp-api/src/main/scala/code/api/util/BerlinGroupCheck.scala index 14e5fae6fa..49ec6cf54f 100644 --- a/obp-api/src/main/scala/code/api/util/BerlinGroupCheck.scala +++ b/obp-api/src/main/scala/code/api/util/BerlinGroupCheck.scala @@ -57,8 +57,7 @@ object BerlinGroupCheck extends MdcLoggable { .toList.filterNot(_.isEmpty) def hasUnwantedConsentIdHeaderForBGEndpoint(path: String, reqHeaders: List[HTTPParam]): Boolean = { - val headerMap: Map[String, HTTPParam] = reqHeaders.map(h => h.name.toLowerCase -> h).toMap - val hasConsentIdId = headerMap.get(RequestHeader.`Consent-ID`.toLowerCase).flatMap(_.values.headOption).isDefined + val hasConsentIdId = RequestHeadersUtil.find(reqHeaders, RequestHeader.`Consent-ID`).flatMap(_.values.headOption).isDefined val parts = path.stripPrefix("/").stripSuffix("/").split("/").toList val doesNotRequireConsentId = parts.reverse match { @@ -79,18 +78,17 @@ object BerlinGroupCheck extends MdcLoggable { forwardResult: (Box[User], Option[CallContext]) ): (Box[User], Option[CallContext]) = { - val headerMap: Map[String, HTTPParam] = reqHeaders.map(h => h.name.toLowerCase -> h).toMap - val maybeRequestId: Option[String] = headerMap.get(RequestHeader.`X-Request-ID`.toLowerCase).flatMap(_.values.headOption) + val maybeRequestId: Option[String] = RequestHeadersUtil.find(reqHeaders, RequestHeader.`X-Request-ID`).flatMap(_.values.headOption) val missingHeaders: List[String] = { if (url.contains(ConstantsBG.berlinGroupVersion1.urlPrefix) && url.endsWith("/consents")) - (berlinGroupMandatoryHeaders ++ berlinGroupMandatoryHeaderConsent).filterNot(headerMap.contains) + (berlinGroupMandatoryHeaders ++ berlinGroupMandatoryHeaderConsent).filterNot(RequestHeadersUtil.exists(reqHeaders, _)) else - berlinGroupMandatoryHeaders.filterNot(headerMap.contains) + berlinGroupMandatoryHeaders.filterNot(RequestHeadersUtil.exists(reqHeaders, _)) } val resultWithWrongDateHeaderCheck: Option[(Box[User], Option[CallContext])] = { - val date: Option[String] = headerMap.get(RequestHeader.Date.toLowerCase).flatMap(_.values.headOption) + val date: Option[String] = RequestHeadersUtil.find(reqHeaders, RequestHeader.Date).flatMap(_.values.headOption) if (date.isDefined && !DateTimeUtil.isValidRfc7231Date(date.get)) { val message = ErrorMessages.NotValidRfc7231Date Some( @@ -155,7 +153,7 @@ object BerlinGroupCheck extends MdcLoggable { // === Signature Header Parsing === val resultWithInvalidSignatureHeaderCheck: Option[(Box[User], Option[CallContext])] = { - val maybeSignature: Option[String] = headerMap.get("signature").flatMap(_.values.headOption) + val maybeSignature: Option[String] = RequestHeadersUtil.find(reqHeaders, RequestHeader.Signature).flatMap(_.values.headOption) maybeSignature.flatMap { header => BerlinGroupSignatureHeaderParser.parseSignatureHeader(header) match { case Right(parsed) => @@ -241,7 +239,7 @@ object BerlinGroupCheck extends MdcLoggable { */ def isTppRequestsWithoutPsuInvolvement(requestHeaders: List[HTTPParam]): Boolean = { def valueOf(name: String): Option[String] = - requestHeaders.find(_.name.equalsIgnoreCase(name)).map(_.values.mkString.trim).filter(_.nonEmpty) + RequestHeadersUtil.find(requestHeaders, name).map(_.values.mkString.trim).filter(_.nonEmpty) val psuIpAddress = valueOf(RequestHeader.`PSU-IP-Address`) val markedAsUnattended = psuIpAddress.contains("0.0.0.0") || diff --git a/obp-api/src/main/scala/code/api/util/BerlinGroupSigning.scala b/obp-api/src/main/scala/code/api/util/BerlinGroupSigning.scala index 6727f2d13f..e049b5ff6a 100644 --- a/obp-api/src/main/scala/code/api/util/BerlinGroupSigning.scala +++ b/obp-api/src/main/scala/code/api/util/BerlinGroupSigning.scala @@ -186,9 +186,9 @@ object BerlinGroupSigning extends MdcLoggable { */ def verifySignedRequest(body: Box[String], verb: String, url: String, reqHeaders: List[HTTPParam], forwardResult: (Box[User], Option[CallContext])): (Box[User], Option[CallContext]) = { def checkRequestIsSigned(requestHeaders: List[HTTPParam]): Boolean = { - requestHeaders.exists(_.name.toLowerCase() == RequestHeader.`TPP-Signature-Certificate`.toLowerCase()) && - requestHeaders.exists(_.name.toLowerCase() == RequestHeader.Signature.toLowerCase()) && - requestHeaders.exists(_.name.toLowerCase() == RequestHeader.Digest.toLowerCase()) + RequestHeadersUtil.exists(requestHeaders, RequestHeader.`TPP-Signature-Certificate`) && + RequestHeadersUtil.exists(requestHeaders, RequestHeader.Signature) && + RequestHeadersUtil.exists(requestHeaders, RequestHeader.Digest) } checkRequestIsSigned(forwardResult._2.map(_.requestHeaders).getOrElse(Nil)) match { case false => @@ -240,7 +240,7 @@ object BerlinGroupSigning extends MdcLoggable { } def getHeaderValue(name: String, requestHeaders: List[HTTPParam]): String = { - requestHeaders.find(_.name.toLowerCase() == name.toLowerCase()).map(_.values.mkString) + RequestHeadersUtil.find(requestHeaders, name).map(_.values.mkString) .getOrElse(SecureRandomUtil.csprng.nextLong().toString) } /** @@ -260,8 +260,7 @@ object BerlinGroupSigning extends MdcLoggable { * ~half of calls where the random Long happened to be negative. */ def getCertificateFromTppSignatureCertificate(requestHeaders: List[HTTPParam]): Box[X509Certificate] = { - requestHeaders - .find(_.name.equalsIgnoreCase(RequestHeader.`TPP-Signature-Certificate`)) + RequestHeadersUtil.find(requestHeaders, RequestHeader.`TPP-Signature-Certificate`) .map(_.values.mkString.trim) .filter(_.nonEmpty) match { case None => diff --git a/obp-api/src/main/scala/code/api/util/ConsentUtil.scala b/obp-api/src/main/scala/code/api/util/ConsentUtil.scala index 40f78eb258..d091622ccb 100644 --- a/obp-api/src/main/scala/code/api/util/ConsentUtil.scala +++ b/obp-api/src/main/scala/code/api/util/ConsentUtil.scala @@ -237,10 +237,7 @@ object Consent extends MdcLoggable { * @return the Consumer-Key value from a Request Header as a String */ def getConsumerKey(requestHeaders: List[HTTPParam]): Option[String] = { - requestHeaders.toSet.filter(_.name.equalsIgnoreCase(RequestHeader.`Consumer-Key`)).toList match { - case x :: Nil => Some(x.values.mkString(", ")) - case _ => None - } + RequestHeadersUtil.findSingle(requestHeaders, RequestHeader.`Consumer-Key`).map(_.values.mkString(", ")) } /** @@ -1552,12 +1549,12 @@ object Consent extends MdcLoggable { // Collect optional headers val headers = callContext.map(_.requestHeaders).getOrElse(Nil) - val tppRedirectUri = headers.find(_.name.equalsIgnoreCase(RequestHeader.`TPP-Redirect-URI`)) - val tppNokRedirectUri = headers.find(_.name.equalsIgnoreCase(RequestHeader.`TPP-Nok-Redirect-URI`)) - val xRequestId = headers.find(_.name.equalsIgnoreCase(RequestHeader.`X-Request-ID`)) - val psuDeviceId = headers.find(_.name.equalsIgnoreCase(RequestHeader.`PSU-Device-ID`)) - val psuIpAddress = headers.find(_.name.equalsIgnoreCase(RequestHeader.`PSU-IP-Address`)) - val psuGeoLocation = headers.find(_.name.equalsIgnoreCase(RequestHeader.`PSU-Geo-Location`)) + val tppRedirectUri = RequestHeadersUtil.find(headers, RequestHeader.`TPP-Redirect-URI`) + val tppNokRedirectUri = RequestHeadersUtil.find(headers, RequestHeader.`TPP-Nok-Redirect-URI`) + val xRequestId = RequestHeadersUtil.find(headers, RequestHeader.`X-Request-ID`) + val psuDeviceId = RequestHeadersUtil.find(headers, RequestHeader.`PSU-Device-ID`) + val psuIpAddress = RequestHeadersUtil.find(headers, RequestHeader.`PSU-IP-Address`) + val psuGeoLocation = RequestHeadersUtil.find(headers, RequestHeader.`PSU-Geo-Location`) def sequenceBoxes[A](boxes: List[Box[A]]): Box[List[A]] = { boxes.foldRight(Full(Nil): Box[List[A]]) { (box, acc) => diff --git a/obp-api/src/main/scala/code/api/util/JwsUtil.scala b/obp-api/src/main/scala/code/api/util/JwsUtil.scala index c885dc3e9b..7e54ae4a28 100644 --- a/obp-api/src/main/scala/code/api/util/JwsUtil.scala +++ b/obp-api/src/main/scala/code/api/util/JwsUtil.scala @@ -73,7 +73,7 @@ object JwsUtil extends MdcLoggable { json.parse(s).extractOpt[JwsProtectedHeader] match { case Some(header) => val headers = header.sigD.pars.flatMap( i => - requestHeaders.find(_.name.equalsIgnoreCase(i)).map(i => s"${i.name.toLowerCase()}: ${i.values.mkString}") + RequestHeadersUtil.find(requestHeaders, i).map(i => s"${i.name.toLowerCase()}: ${i.values.mkString}") ) val requestTarget = s"""(request-target): ${verb.toLowerCase()} ${url}\n""" requestTarget + headers.mkString("\n") + "\n" // Add new line after each item @@ -100,14 +100,14 @@ object JwsUtil extends MdcLoggable { headerValue == s"SHA-256=${computeDigest(httpBody)}" } def getDigestHeaderValue(requestHeaders: List[HTTPParam]): String = { - requestHeaders.find(_.name.equalsIgnoreCase("digest")).map(_.values.mkString).getOrElse("None") + RequestHeadersUtil.find(requestHeaders, "digest").map(_.values.mkString).getOrElse("None") } def getJwsHeaderValue(requestHeaders: List[HTTPParam]): String = { - requestHeaders.find(_.name.equalsIgnoreCase("x-jws-signature")).map(_.values.mkString).getOrElse("None") + RequestHeadersUtil.find(requestHeaders, "x-jws-signature").map(_.values.mkString).getOrElse("None") } def checkRequestIsSigned(requestHeaders: List[HTTPParam]): Boolean = { - requestHeaders.exists(_.name.equalsIgnoreCase("x-jws-signature")) || - requestHeaders.exists(_.name.equalsIgnoreCase("digest")) + RequestHeadersUtil.exists(requestHeaders, "x-jws-signature") || + RequestHeadersUtil.exists(requestHeaders, "digest") } private def getDeferredCriticalHeaders() = { val deferredCriticalHeaders = new util.HashSet[String]() diff --git a/obp-api/src/main/scala/code/api/util/RequestHeadersUtil.scala b/obp-api/src/main/scala/code/api/util/RequestHeadersUtil.scala index 2d50db5ec6..75d282283c 100644 --- a/obp-api/src/main/scala/code/api/util/RequestHeadersUtil.scala +++ b/obp-api/src/main/scala/code/api/util/RequestHeadersUtil.scala @@ -30,7 +30,45 @@ package code.api.util import code.api.RequestHeader._ import code.api.util.APIUtil.HTTPParam +/** + * This object is the one place OBP looks up a request header by name. + * + * HTTP header names are case-insensitive (RFC 9110, section 5.1), and over HTTP/2 every name arrives + * in lower case, so a client sending `Consent-JWT` may reach OBP as `consent-jwt`. The http4s request + * already treats names that way, but `Http4sCallContextBuilder` copies the headers into a + * `List[HTTPParam]` with the names exactly as received, and a lookup written as `_.name == "Consent-JWT"` + * then misses the header. Every lookup in a request header list goes through the functions below, + * which match names ignoring case; HeaderLookupConventionsTest fails the build on one that does not. + */ object RequestHeadersUtil { + + /** This returns true when the header has the given name, ignoring letter case. */ + def isNamed(header: HTTPParam, name: String): Boolean = + header != null && header.name != null && header.name.equalsIgnoreCase(name) + + /** This returns true when any header has the given name. */ + def exists(requestHeaders: List[HTTPParam], name: String): Boolean = + requestHeaders.exists(isNamed(_, name)) + + /** This returns the first header with the given name. */ + def find(requestHeaders: List[HTTPParam], name: String): Option[HTTPParam] = + requestHeaders.find(isNamed(_, name)) + + /** + * This returns the header with the given name only when exactly one distinct header has it, and + * None when there is none or there are several that differ. Identical repeats count once. It is for + * credentials such as Consent-JWT, where a request carrying two different values is ambiguous. + */ + def findSingle(requestHeaders: List[HTTPParam], name: String): Option[HTTPParam] = + requestHeaders.toSet.filter(isNamed(_, name)).toList match { + case header :: Nil => Some(header) + case _ => None + } + + /** This groups the headers by name, ignoring letter case; the keys are the names in lower case. */ + def groupByName(requestHeaders: List[HTTPParam]): Map[String, List[HTTPParam]] = + requestHeaders.groupBy(_.name.toLowerCase(java.util.Locale.ROOT)) + def checkEmptyRequestHeaderValues(requestHeaders: List[HTTPParam]): List[String] = { val emptyValues = requestHeaders .filter(header => header != null && (header.values == null || header.values.isEmpty || header.values.exists(_.trim.isEmpty))) diff --git a/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala b/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala index feaa38add7..eb7a76e5a0 100644 --- a/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala +++ b/obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala @@ -149,7 +149,7 @@ object WriteMetricUtil extends MdcLoggable { } private def requestHeaderValue(cc: CallContextLight, headerName: String): String = - cc.requestHeaders.find(_.name.equalsIgnoreCase(headerName)).map(_.values.mkString(",")).getOrElse("") + RequestHeadersUtil.find(cc.requestHeaders, headerName).map(_.values.mkString(",")).getOrElse("") private def saveMetricSafely(cc: CallContextLight, fields: MetricFields): Unit = { import fields._ diff --git a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala index 23e0aecf70..0cda869526 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/Http4s510.scala @@ -494,7 +494,7 @@ object Http4s510 { val df = new java.text.SimpleDateFormat(DateWithSeconds) val headerEpoch: Long = scala.util.Try(df.parse(headerValue).getTime).getOrElse(0L) val requestHeaders = cc.requestHeaders - .filter(i => i.name == "limit" || i.name == "offset").sortBy(_.name) + .filter(i => code.api.util.RequestHeadersUtil.isNamed(i, "limit") || code.api.util.RequestHeadersUtil.isNamed(i, "offset")).sortBy(_.name) val hashedRequestPayload = code.api.util.HashUtil.Sha256Hash(cc.url + requestHeaders) val consumerId = cc.consumer.map(_.consumerId.get).getOrElse("None") val userId = scala.util.Try(cc.userId).getOrElse("None") diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index df26694f89..bc3aee14d3 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -5005,8 +5005,7 @@ object Http4s600 { // but reads from CallContext.requestHeaders (populated by the http4s context builder) instead of // Lift's thread-local S.request. private def parseDirectLoginParams(cc: CallContext): Map[String, String] = { - def find(name: String): Option[String] = cc.requestHeaders - .find(_.name.equalsIgnoreCase(name)) + def find(name: String): Option[String] = code.api.util.RequestHeadersUtil.find(cc.requestHeaders, name) .flatMap(_.values.headOption) val directLoginHeader = find("DirectLogin") val authHeader = find("Authorization") diff --git a/obp-api/src/test/scala/code/api/util/ConsentHeaderDispatchTest.scala b/obp-api/src/test/scala/code/api/util/ConsentHeaderDispatchTest.scala new file mode 100644 index 0000000000..ff42db4286 --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/ConsentHeaderDispatchTest.scala @@ -0,0 +1,57 @@ +package code.api.util + +import code.api.RequestHeader +import code.api.util.APIUtil.HTTPParam +import code.setup.ServerSetup +import net.liftweb.common.Failure + +import scala.concurrent.Await +import scala.concurrent.duration._ +import scala.util.Try + +/** + * This class tests which consent scheme authenticates a request that carries a consent id. + * + * OBP's own consent header is `Consent-Id`, and Berlin Group's is `Consent-ID`. The two names differ + * only in letter case, and HTTP header names are case-insensitive (over HTTP/2 both arrive as + * `consent-id`), so the name cannot tell the schemes apart. The request path does: + * `APIUtil.getUserAndSessionContextFuture` applies Berlin Group's consent rules only on a Berlin + * Group path, and OBP's everywhere else, whichever spelling the client used. + * + * Each scenario sends a consent id that matches no consent and reads which scheme refused it. OBP's + * rules refuse it as `ConsentHeaderValueInvalid` (neither a known consent id nor a JWT); Berlin + * Group's refuse it as `ConsentNotFound`. + */ +class ConsentHeaderDispatchTest extends ServerSetup { + + private val obpPath = "/obp/v5.1.0/users/current" + private val berlinGroupPath = "/berlin-group/v1.3/accounts" + private val unknownConsentId = "no-such-consent" + + /** This runs authentication for a GET of `path` with one header, and returns the error message. */ + private def errorFor(path: String, headerName: String): String = { + setPropsValues("consents.allowed" -> "true") + val callContext = CallContext(url = path, verb = "GET", requestHeaders = List(HTTPParam(headerName, List(unknownConsentId)))) + Try(Await.result(APIUtil.getUserAndSessionContextFuture(callContext), 30.seconds)).map(_._1) match { + case scala.util.Success(Failure(message, _, _)) => message + case scala.util.Success(other) => fail(s"expected the consent to be refused, got $other") + case scala.util.Failure(exception) => exception.getMessage + } + } + + feature("The request path, not the header's letter case, picks the consent scheme") { + + List(RequestHeader.`Consent-Id`, RequestHeader.`Consent-ID`, "consent-id").foreach { headerName => + scenario(s"On an OBP path, a $headerName header is checked by OBP's consent rules") { + val message = errorFor(obpPath, headerName) + message should include(ErrorMessages.ConsentHeaderValueInvalid) + } + + scenario(s"On a Berlin Group path, a $headerName header is checked by Berlin Group's consent rules") { + val message = errorFor(berlinGroupPath, headerName) + message should include(ErrorMessages.ConsentNotFound) + message should include(unknownConsentId) + } + } + } +} diff --git a/obp-api/src/test/scala/code/api/util/HeaderLookupConventionsTest.scala b/obp-api/src/test/scala/code/api/util/HeaderLookupConventionsTest.scala new file mode 100644 index 0000000000..9a19544b6a --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/HeaderLookupConventionsTest.scala @@ -0,0 +1,80 @@ +package code.api.util + +import java.io.File + +import org.scalatest.{FeatureSpec, Matchers} + +import scala.io.Source + +/** + * This class keeps every request header lookup going through RequestHeadersUtil. + * + * HTTP header names are case-insensitive, and over HTTP/2 every name arrives in lower case. A lookup + * written as `requestHeaders.find(_.name == "Consent-JWT")` works in a test that sends `Consent-JWT` + * and silently misses the header from a real HTTP/2 client. RequestHeadersUtil matches names ignoring + * case; this suite reads the main sources and fails on a lookup that compares a header name itself. + * + * It is a text scan, so it recognises a header list by its variable name: `requestHeaders`, + * `reqHeaders` or `headers`, on the line of the comparison or one of the two lines before it (a + * lookup is often split across lines). It does not need a database or a server. + */ +class HeaderLookupConventionsTest extends FeatureSpec with Matchers { + + // Maven runs the suite from obp-api/, an IDE often from the repository root. + private val sourceRoot: File = + List(new File("src/main/scala"), new File("obp-api/src/main/scala")).find(_.isDirectory) + .getOrElse(throw new IllegalStateException("HeaderLookupConventionsTest cannot find obp-api/src/main/scala")) + + private val headerList = """\b(requestHeaders|reqHeaders|headers)\b""".r + + /** A comparison of a name itself: `==` or `!=`, equals or equalsIgnoreCase, a lower-cased name used as a key, or a pattern binding the name. */ + private val nameComparison = + """\.name\s*(==|!=)|\.name\.(equalsIgnoreCase|equals)\(|\.name\.toLowerCase(\([^)]*\))?\s*(==|->)|groupBy\(_\.name|HTTPParam\(\s*name\s*,""".r + + /** This returns the 1-based numbers of the lines in `text` that compare a request header name directly. */ + private def directLookups(text: String): List[Int] = { + val lines = text.split("\n", -1).toList + lines.indices.toList.filter { index => + nameComparison.findFirstIn(lines(index)).isDefined && + headerList.findFirstIn(lines.slice(math.max(0, index - 2), index + 1).mkString(" ")).isDefined + }.map(_ + 1) + } + + private def scalaFiles(dir: File): List[File] = + Option(dir.listFiles).toList.flatten.flatMap(file => if (file.isDirectory) scalaFiles(file) else List(file)) + .filter(_.getName.endsWith(".scala")) + + private def read(file: File): String = { + val source = Source.fromFile(file, "UTF-8") + try source.mkString finally source.close() + } + + feature("Request headers are looked up through RequestHeadersUtil") { + + scenario("The scan recognises a direct lookup and passes one through the helper") { + directLookups("""requestHeaders.find(_.name == "Consent-JWT")""") shouldBe List(1) + directLookups("""reqHeaders.exists(_.name.equalsIgnoreCase(DAuthHeaderKey))""") shouldBe List(1) + directLookups("val raw = requestHeaders\n .find(_.name.toLowerCase() == name.toLowerCase())") shouldBe List(2) + directLookups("""val byName = reqHeaders.map(h => h.name.toLowerCase -> h).toMap""") shouldBe List(1) + directLookups("""callContext.requestHeaders.groupBy(_.name.toLowerCase)""") shouldBe List(1) + directLookups("requestHeaders.collectFirst {\n case HTTPParam(name, value :: _) if name == \"Force-Error\" => value") shouldBe List(2) + + directLookups("""RequestHeadersUtil.find(requestHeaders, "Consent-JWT")""") shouldBe Nil + directLookups("""requestHeaders.map(header => header.name + ": " + header.values.mkString)""") shouldBe Nil + directLookups("""attributes.find(_.name == "CERTIFICATE_CA_NAME")""") shouldBe Nil + } + + scenario("No main source compares a request header name itself") { + val files = scalaFiles(sourceRoot).filterNot(_.getName == "RequestHeadersUtil.scala") + withClue("the scan must be reading the main sources: ") { files.size should be > 1000 } + val found = files.flatMap { file => + val path = sourceRoot.toPath.relativize(file.toPath).toString + directLookups(read(file)).map(line => s"$path:$line") + }.sorted + withClue("These lines compare a request header name directly. Use RequestHeadersUtil.find, exists, " + + "findSingle or isNamed, which ignore letter case as HTTP requires:\n" + found.mkString("\n") + "\n") { + found shouldBe empty + } + } + } +}