From 2097518a202d57abdbac6c561cbdea825eadfab2 Mon Sep 17 00:00:00 2001 From: hongwei Date: Tue, 24 Oct 2023 14:49:56 +0200 Subject: [PATCH 1/2] bugfix/set the example value to constant in dynamic endpoints --- .../api/dynamic/entity/helper/DynamicEntityHelper.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala index 972ccc47a0..6df2c496f5 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala @@ -1,6 +1,6 @@ package code.api.dynamic.entity.helper -import code.api.util.APIUtil.{EmptyBody, ResourceDoc, authenticationRequiredMessage, generateUUID} +import code.api.util.APIUtil.{EmptyBody, ResourceDoc, authenticationRequiredMessage} import code.api.util.ApiRole.getOrCreateDynamicApiRole import code.api.util.ApiTag._ import code.api.util.ErrorMessages.{InvalidJsonFormat, UnknownError, UserHasMissingRoles, UserNotLoggedIn} @@ -577,10 +577,10 @@ case class DynamicEntityInfo(definition: String, entityName: String, bankId: Opt val bankIdJObject: JObject = ("bank-id" -> ExampleValue.bankIdExample.value) def getSingleExample: JObject = if (bankId.isDefined){ - val SingleObject: JObject = (singleName -> (JObject(JField(idName, JString(generateUUID())) :: getSingleExampleWithoutId.obj))) + val SingleObject: JObject = (singleName -> (JObject(JField(idName, JString(ExampleValue.idExample.value)) :: getSingleExampleWithoutId.obj))) bankIdJObject merge SingleObject } else{ - (singleName -> (JObject(JField(idName, JString(generateUUID())) :: getSingleExampleWithoutId.obj))) + (singleName -> (JObject(JField(idName, JString(ExampleValue.idExample.value)) :: getSingleExampleWithoutId.obj))) } def getExampleList: JObject = if (bankId.isDefined){ From 8b8931be1c8ed1fb4098bd66dc2485e7011f8d0e Mon Sep 17 00:00:00 2001 From: hongwei Date: Tue, 24 Oct 2023 15:34:49 +0200 Subject: [PATCH 2/2] bugfix/use scalaCache instead of resourceDocshashMap and added debug log for hashmap --- .../ResourceDocsAPIMethods.scala | 2 + .../main/scala/code/api/util/ApiRole.scala | 5 +- .../scala/code/api/util/DynamicUtil.scala | 4 +- .../code/api/v1_4_0/JSONFactory1_4_0.scala | 159 ++++++++++-------- 4 files changed, 99 insertions(+), 71 deletions(-) diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala index 3fc8605bb4..1743b6c292 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala @@ -12,6 +12,7 @@ import code.api.dynamic.entity.helper.DynamicEntityHelper import code.api.util.APIUtil._ import code.api.util.ApiRole.{canReadDynamicResourceDocsAtOneBank, canReadResourceDoc, canReadStaticResourceDoc} import code.api.util.ApiTag._ +import code.api.util.DynamicUtil.{dynamicCompileResult, logger} import code.api.util.ExampleValue.endpointMappingRequestBodyExample import code.api.util._ import code.api.v1_4_0.JSONFactory1_4_0.ResourceDocsJson @@ -82,6 +83,7 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth private val specialInstructionMap = new ConcurrentHashMap[String, Option[String]]() // Find any special instructions for partialFunctionName def getSpecialInstructions(partialFunctionName: String): Option[String] = { + logger.debug(s"ResourceDocsAPIMethods.getSpecialInstructions.specialInstructionMap.size is ${specialInstructionMap.size()}") specialInstructionMap.computeIfAbsent(partialFunctionName, _ => { // The files should be placed in a folder called special_instructions_for_resources folder inside the src resources folder // Each file should match a partial function name or it will be ignored. diff --git a/obp-api/src/main/scala/code/api/util/ApiRole.scala b/obp-api/src/main/scala/code/api/util/ApiRole.scala index 439ec4c8a8..d6a40c1521 100644 --- a/obp-api/src/main/scala/code/api/util/ApiRole.scala +++ b/obp-api/src/main/scala/code/api/util/ApiRole.scala @@ -5,6 +5,7 @@ import code.api.dynamic.endpoint.helper.DynamicEndpointHelper import java.util.concurrent.ConcurrentHashMap import code.api.dynamic.endpoint.helper.DynamicEndpointHelper import code.api.dynamic.entity.helper.DynamicEntityHelper +import code.util.Helper.MdcLoggable import com.openbankproject.commons.util.{JsonAble, ReflectUtils} import net.liftweb.json.{Formats, JsonAST} import net.liftweb.json.JsonDSL._ @@ -63,7 +64,7 @@ object RoleCombination { // Remember to add to the list of roles below -object ApiRole { +object ApiRole extends MdcLoggable{ case class CanSearchWarehouse(requiresBankId: Boolean = false) extends ApiRole lazy val canSearchWarehouse = CanSearchWarehouse() @@ -979,9 +980,11 @@ object ApiRole { } def getOrCreateDynamicApiRole(roleName: String, requiresBankId: Boolean = false): ApiRole = { + logger.debug(s"code.api.util.ApiRole.getOrCreateDynamicApiRole.size is ${dynamicApiRoles.size()}") dynamicApiRoles.computeIfAbsent(roleName, _ => DynamicApiRole(roleName, requiresBankId)) } def removeDynamicApiRole(roleName: String): ApiRole = { + logger.debug(s"code.api.util.ApiRole.removeDynamicApiRole.size is ${dynamicApiRoles.size()}") dynamicApiRoles.remove(roleName) } diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala index e3080067e1..96a1a6ed6a 100644 --- a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala @@ -2,6 +2,7 @@ package code.api.util import code.api.{APIFailureNewStyle, JsonResponseException} import code.api.util.ErrorMessages.DynamicResourceDocMethodDependency +import code.util.Helper.MdcLoggable import com.openbankproject.commons.model.BankId import com.openbankproject.commons.util.Functions.Memo import com.openbankproject.commons.util.{JsonUtils, ReflectUtils} @@ -26,7 +27,7 @@ import scala.reflect.runtime.universe.runtimeMirror import scala.runtime.NonLocalReturnControl import scala.tools.reflect.{ToolBox, ToolBoxError} -object DynamicUtil { +object DynamicUtil extends MdcLoggable{ val toolBox: ToolBox[universe.type] = runtimeMirror(getClass.getClassLoader).mkToolBox() private val memoClassPool = new Memo[ClassLoader, ClassPool] @@ -50,6 +51,7 @@ object DynamicUtil { * @return compiled Full[function|object|class] or Failure */ def compileScalaCode[T](code: String): Box[T] = { + logger.debug(s"code.api.util.DynamicUtil.compileScalaCode.size is ${dynamicCompileResult.size()}") val compiledResult: Box[Any] = dynamicCompileResult.computeIfAbsent(code, _ => { val tree = try { toolBox.parse(code) diff --git a/obp-api/src/main/scala/code/api/v1_4_0/JSONFactory1_4_0.scala b/obp-api/src/main/scala/code/api/v1_4_0/JSONFactory1_4_0.scala index 0245756eb0..26e2ce142f 100644 --- a/obp-api/src/main/scala/code/api/v1_4_0/JSONFactory1_4_0.scala +++ b/obp-api/src/main/scala/code/api/v1_4_0/JSONFactory1_4_0.scala @@ -1,6 +1,7 @@ package code.api.v1_4_0 import code.api.berlin.group.v1_3.JvalueCaseClass +import code.api.cache.Caching import java.util.Date import code.api.util.APIUtil.{EmptyBody, PrimaryDataBody, ResourceDoc} @@ -21,10 +22,12 @@ import net.liftweb.json.{Formats, JDouble, JInt, JString} import net.liftweb.json.JsonAST.{JArray, JBool, JNothing, JObject, JValue} import net.liftweb.util.StringHelpers import code.util.Helper.MdcLoggable +import com.tesobe.CacheKeyFromArguments import org.apache.commons.lang3.StringUtils -import java.util.concurrent.ConcurrentHashMap import java.util.regex.Pattern import java.lang.reflect.Field +import java.util.UUID.randomUUID +import scala.concurrent.duration._ object JSONFactory1_4_0 extends MdcLoggable{ implicit def formats: Formats = CustomJsonFormats.formats @@ -515,79 +518,97 @@ object JSONFactory1_4_0 extends MdcLoggable{ jsonFieldsDescription.mkString(jsonTitleType,"","\n") } - private val createResourceDocJsonMemo = new ConcurrentHashMap[ResourceDoc, ResourceDocJson] + val createResourceDocJsonTTL : Int = APIUtil.getPropsValue(s"createResourceDocJson.cache.ttl.seconds", "86400").toInt + + def createResourceDocJsonCached(resourceDocUpdatedTags: ResourceDoc, + isVersion4OrHigher:Boolean, locale: Option[String], + urlParametersI18n:String , + jsonRequestBodyFieldsI18n:String, + jsonResponseBodyFieldsI18n:String + ): ResourceDocJson = { + /** + * Please note that "var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)" + * is just a temporary value field with UUID values in order to prevent any ambiguity. + * The real value will be assigned by Macro during compile time at this line of a code: + * https://github.com/OpenBankProject/scala-macros/blob/master/macros/src/main/scala/com/tesobe/CacheKeyFromArgumentsMacro.scala#L49 + */ + var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString) + CacheKeyFromArguments.buildCacheKey { + Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(createResourceDocJsonTTL second) { + // There are multiple flavours of markdown. For instance, original markdown emphasises underscores (surrounds _ with ()) + // But we don't want to have to escape underscores (\_) in our documentation + // Thus we use a flavour of markdown that ignores underscores in words. (Github markdown does this too) + // We return html rather than markdown to the consumer so they don't have to bother with these questions. + + //Here area some endpoints, which should not be added the description: + // 1st: Dynamic entity endpoint, + // 2rd: Dynamic endpoint endpoints, + // 3rd: all the user created endpoints, + val fieldsDescription = + if (resourceDocUpdatedTags.tags.toString.contains("Dynamic-Entity") + || resourceDocUpdatedTags.tags.toString.contains("Dynamic-Endpoint") + || resourceDocUpdatedTags.roles.toString.contains("DynamicEntity") + || resourceDocUpdatedTags.roles.toString.contains("DynamicEntities") + || resourceDocUpdatedTags.roles.toString.contains("DynamicEndpoint")) { + "" + } else { + //1st: prepare the description from URL + val urlParametersDescription: String = prepareUrlParameterDescription(resourceDocUpdatedTags.requestUrl, urlParametersI18n) + //2rd: get the fields description from the post json body: + val exampleRequestBodyFieldsDescription = + if (resourceDocUpdatedTags.requestVerb == "POST") { + prepareJsonFieldDescription(resourceDocUpdatedTags.exampleRequestBody, "request", jsonRequestBodyFieldsI18n, jsonResponseBodyFieldsI18n) + } else { + "" + } + //3rd: get the fields description from the response body: + //response body can be a nest class, need to loop all the fields. + val responseFieldsDescription = prepareJsonFieldDescription(resourceDocUpdatedTags.successResponseBody, "response", jsonRequestBodyFieldsI18n, jsonResponseBodyFieldsI18n) + urlParametersDescription ++ exampleRequestBodyFieldsDescription ++ responseFieldsDescription + } + val resourceDocDescription = I18NUtil.ResourceDocTranslation.translate( + I18NResourceDocField.DESCRIPTION, + resourceDocUpdatedTags.operationId, + locale, + resourceDocUpdatedTags.description.stripMargin.trim + ) + val description = resourceDocDescription ++ fieldsDescription + val summary = resourceDocUpdatedTags.summary.replaceFirst("""\.(\s*)$""", "$1") // remove the ending dot in summary + val translatedSummary = I18NUtil.ResourceDocTranslation.translate(I18NResourceDocField.SUMMARY, resourceDocUpdatedTags.operationId, locale, summary) + + ResourceDocJson( + operation_id = resourceDocUpdatedTags.operationId, + request_verb = resourceDocUpdatedTags.requestVerb, + request_url = resourceDocUpdatedTags.requestUrl, + summary = translatedSummary, + // Strip the margin character (|) and line breaks and convert from markdown to html + description = PegdownOptions.convertPegdownToHtmlTweaked(description), //.replaceAll("\n", ""), + description_markdown = description, + example_request_body = resourceDocUpdatedTags.exampleRequestBody, + success_response_body = resourceDocUpdatedTags.successResponseBody, + error_response_bodies = resourceDocUpdatedTags.errorResponseBodies, + implemented_by = ImplementedByJson(resourceDocUpdatedTags.implementedInApiVersion.fullyQualifiedVersion, resourceDocUpdatedTags.partialFunctionName), // was resourceDocUpdatedTags.implementedInApiVersion.noV + tags = resourceDocUpdatedTags.tags.map(i => i.tag), + typed_request_body = createTypedBody(resourceDocUpdatedTags.exampleRequestBody), + typed_success_response_body = createTypedBody(resourceDocUpdatedTags.successResponseBody), + roles = resourceDocUpdatedTags.roles, + is_featured = resourceDocUpdatedTags.isFeatured, + special_instructions = PegdownOptions.convertPegdownToHtmlTweaked(resourceDocUpdatedTags.specialInstructions.getOrElse("").stripMargin), + specified_url = resourceDocUpdatedTags.specifiedUrl.getOrElse(""), + connector_methods = resourceDocUpdatedTags.connectorMethods, + created_by_bank_id = if (isVersion4OrHigher) resourceDocUpdatedTags.createdByBankId else None // only for V400 we show the bankId + ) + } + } + } + def createResourceDocJson(rd: ResourceDoc, isVersion4OrHigher:Boolean, locale: Option[String], urlParametersI18n:String ,jsonRequestBodyFieldsI18n:String, jsonResponseBodyFieldsI18n:String) : ResourceDocJson = { // We MUST recompute all resource doc values due to translation via Web UI props val endpointTags = getAllEndpointTagsBox(rd.operationId).map(endpointTag =>ResourceDocTag(endpointTag.tagName)) - val resourceDocUpdatedTags: ResourceDoc = rd.copy(tags = endpointTags++ rd.tags) - logger.debug(s"createResourceDocJson createResourceDocJsonMemo.size is ${createResourceDocJsonMemo.size()}") - createResourceDocJsonMemo.compute(resourceDocUpdatedTags, (k, v) => { - // There are multiple flavours of markdown. For instance, original markdown emphasises underscores (surrounds _ with ()) - // But we don't want to have to escape underscores (\_) in our documentation - // Thus we use a flavour of markdown that ignores underscores in words. (Github markdown does this too) - // We return html rather than markdown to the consumer so they don't have to bother with these questions. - - //Here area some endpoints, which should not be added the description: - // 1st: Dynamic entity endpoint, - // 2rd: Dynamic endpoint endpoints, - // 3rd: all the user created endpoints, - val fieldsDescription = - if(resourceDocUpdatedTags.tags.toString.contains("Dynamic-Entity") - ||resourceDocUpdatedTags.tags.toString.contains("Dynamic-Endpoint") - ||resourceDocUpdatedTags.roles.toString.contains("DynamicEntity") - ||resourceDocUpdatedTags.roles.toString.contains("DynamicEntities") - ||resourceDocUpdatedTags.roles.toString.contains("DynamicEndpoint")) { - "" - } else{ - //1st: prepare the description from URL - val urlParametersDescription: String = prepareUrlParameterDescription(resourceDocUpdatedTags.requestUrl, urlParametersI18n) - //2rd: get the fields description from the post json body: - val exampleRequestBodyFieldsDescription = - if (resourceDocUpdatedTags.requestVerb=="POST" ){ - prepareJsonFieldDescription(resourceDocUpdatedTags.exampleRequestBody,"request", jsonRequestBodyFieldsI18n, jsonResponseBodyFieldsI18n) - } else { - "" - } - //3rd: get the fields description from the response body: - //response body can be a nest class, need to loop all the fields. - val responseFieldsDescription = prepareJsonFieldDescription(resourceDocUpdatedTags.successResponseBody,"response", jsonRequestBodyFieldsI18n, jsonResponseBodyFieldsI18n) - urlParametersDescription ++ exampleRequestBodyFieldsDescription ++ responseFieldsDescription - } + val updatedTagsResourceDoc: ResourceDoc = rd.copy(tags = endpointTags++ rd.tags) - val resourceDocDescription = I18NUtil.ResourceDocTranslation.translate( - I18NResourceDocField.DESCRIPTION, - resourceDocUpdatedTags.operationId, - locale, - resourceDocUpdatedTags.description.stripMargin.trim - ) - val description = resourceDocDescription ++ fieldsDescription - val summary = resourceDocUpdatedTags.summary.replaceFirst("""\.(\s*)$""", "$1") // remove the ending dot in summary - val translatedSummary = I18NUtil.ResourceDocTranslation.translate(I18NResourceDocField.SUMMARY, resourceDocUpdatedTags.operationId, locale, summary) - - ResourceDocJson( - operation_id = resourceDocUpdatedTags.operationId, - request_verb = resourceDocUpdatedTags.requestVerb, - request_url = resourceDocUpdatedTags.requestUrl, - summary = translatedSummary, - // Strip the margin character (|) and line breaks and convert from markdown to html - description = PegdownOptions.convertPegdownToHtmlTweaked(description), //.replaceAll("\n", ""), - description_markdown = description, - example_request_body = resourceDocUpdatedTags.exampleRequestBody, - success_response_body = resourceDocUpdatedTags.successResponseBody, - error_response_bodies = resourceDocUpdatedTags.errorResponseBodies, - implemented_by = ImplementedByJson(resourceDocUpdatedTags.implementedInApiVersion.fullyQualifiedVersion, resourceDocUpdatedTags.partialFunctionName), // was resourceDocUpdatedTags.implementedInApiVersion.noV - tags = resourceDocUpdatedTags.tags.map(i => i.tag), - typed_request_body = createTypedBody(resourceDocUpdatedTags.exampleRequestBody), - typed_success_response_body = createTypedBody(resourceDocUpdatedTags.successResponseBody), - roles = resourceDocUpdatedTags.roles, - is_featured = resourceDocUpdatedTags.isFeatured, - special_instructions = PegdownOptions.convertPegdownToHtmlTweaked(resourceDocUpdatedTags.specialInstructions.getOrElse("").stripMargin), - specified_url = resourceDocUpdatedTags.specifiedUrl.getOrElse(""), - connector_methods = resourceDocUpdatedTags.connectorMethods, - created_by_bank_id= if (isVersion4OrHigher) rd.createdByBankId else None // only for V400 we show the bankId - ) - }) + createResourceDocJsonCached(updatedTagsResourceDoc, isVersion4OrHigher:Boolean, locale: Option[String], urlParametersI18n:String ,jsonRequestBodyFieldsI18n:String, jsonResponseBodyFieldsI18n:String) } def createResourceDocsJson(resourceDocList: List[ResourceDoc], isVersion4OrHigher:Boolean, locale: Option[String]) : ResourceDocsJson = {