Merge pull request #2303 from hongwei1/bugfix/CacheForResourceDocs

Bugfix/cache for resource docs
OpenBankProject · Oct 24, 2023 · 1ec2ee0 · 1ec2ee0
2 parents b9fce7e + 8b8931b
commit 1ec2ee0
Show file tree

Hide file tree

Showing 5 changed files with 102 additions and 74 deletions.
diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala
@@ -12,6 +12,7 @@ import code.api.dynamic.entity.helper.DynamicEntityHelper
 import code.api.util.APIUtil._
 import code.api.util.ApiRole.{canReadDynamicResourceDocsAtOneBank, canReadResourceDoc, canReadStaticResourceDoc}
 import code.api.util.ApiTag._
+import code.api.util.DynamicUtil.{dynamicCompileResult, logger}
 import code.api.util.ExampleValue.endpointMappingRequestBodyExample
 import code.api.util._
 import code.api.v1_4_0.JSONFactory1_4_0.ResourceDocsJson
@@ -82,6 +83,7 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth
     private val specialInstructionMap = new ConcurrentHashMap[String, Option[String]]()
     // Find any special instructions for partialFunctionName
     def getSpecialInstructions(partialFunctionName: String):  Option[String] = {
+      logger.debug(s"ResourceDocsAPIMethods.getSpecialInstructions.specialInstructionMap.size is ${specialInstructionMap.size()}")
       specialInstructionMap.computeIfAbsent(partialFunctionName, _ => {
         // The files should be placed in a folder called special_instructions_for_resources folder inside the src resources folder
         // Each file should match a partial function name or it will be ignored.

diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala
@@ -1,6 +1,6 @@
 package code.api.dynamic.entity.helper
 
-import code.api.util.APIUtil.{EmptyBody, ResourceDoc, authenticationRequiredMessage, generateUUID}
+import code.api.util.APIUtil.{EmptyBody, ResourceDoc, authenticationRequiredMessage}
 import code.api.util.ApiRole.getOrCreateDynamicApiRole
 import code.api.util.ApiTag._
 import code.api.util.ErrorMessages.{InvalidJsonFormat, UnknownError, UserHasMissingRoles, UserNotLoggedIn}
@@ -577,10 +577,10 @@ case class DynamicEntityInfo(definition: String, entityName: String, bankId: Opt
   val bankIdJObject: JObject = ("bank-id" -> ExampleValue.bankIdExample.value)
 
   def getSingleExample: JObject = if (bankId.isDefined){
-    val SingleObject: JObject = (singleName -> (JObject(JField(idName, JString(generateUUID())) :: getSingleExampleWithoutId.obj)))
+    val SingleObject: JObject = (singleName -> (JObject(JField(idName, JString(ExampleValue.idExample.value)) :: getSingleExampleWithoutId.obj)))
     bankIdJObject merge SingleObject
   } else{
-    (singleName -> (JObject(JField(idName, JString(generateUUID())) :: getSingleExampleWithoutId.obj)))
+    (singleName -> (JObject(JField(idName, JString(ExampleValue.idExample.value)) :: getSingleExampleWithoutId.obj)))
   }
 
   def getExampleList: JObject =  if (bankId.isDefined){

diff --git a/obp-api/src/main/scala/code/api/util/ApiRole.scala b/obp-api/src/main/scala/code/api/util/ApiRole.scala
@@ -5,6 +5,7 @@ import code.api.dynamic.endpoint.helper.DynamicEndpointHelper
 import java.util.concurrent.ConcurrentHashMap
 import code.api.dynamic.endpoint.helper.DynamicEndpointHelper
 import code.api.dynamic.entity.helper.DynamicEntityHelper
+import code.util.Helper.MdcLoggable
 import com.openbankproject.commons.util.{JsonAble, ReflectUtils}
 import net.liftweb.json.{Formats, JsonAST}
 import net.liftweb.json.JsonDSL._
@@ -63,7 +64,7 @@ object RoleCombination {
 // Remember to add to the list of roles below
 
 
-object ApiRole {
+object ApiRole extends MdcLoggable{
 
   case class CanSearchWarehouse(requiresBankId: Boolean = false) extends ApiRole
   lazy val canSearchWarehouse = CanSearchWarehouse()
@@ -979,9 +980,11 @@ object ApiRole {
   }
 
   def getOrCreateDynamicApiRole(roleName: String, requiresBankId: Boolean = false): ApiRole = {
+    logger.debug(s"code.api.util.ApiRole.getOrCreateDynamicApiRole.size is ${dynamicApiRoles.size()}")
     dynamicApiRoles.computeIfAbsent(roleName, _ => DynamicApiRole(roleName, requiresBankId))
   }
   def removeDynamicApiRole(roleName: String): ApiRole = {
+    logger.debug(s"code.api.util.ApiRole.removeDynamicApiRole.size is ${dynamicApiRoles.size()}")
     dynamicApiRoles.remove(roleName)
   }
 

diff --git a/obp-api/src/main/scala/code/api/util/DynamicUtil.scala b/obp-api/src/main/scala/code/api/util/DynamicUtil.scala
@@ -2,6 +2,7 @@ package code.api.util
 
 import code.api.{APIFailureNewStyle, JsonResponseException}
 import code.api.util.ErrorMessages.DynamicResourceDocMethodDependency
+import code.util.Helper.MdcLoggable
 import com.openbankproject.commons.model.BankId
 import com.openbankproject.commons.util.Functions.Memo
 import com.openbankproject.commons.util.{JsonUtils, ReflectUtils}
@@ -26,7 +27,7 @@ import scala.reflect.runtime.universe.runtimeMirror
 import scala.runtime.NonLocalReturnControl
 import scala.tools.reflect.{ToolBox, ToolBoxError}
 
-object DynamicUtil {
+object DynamicUtil extends MdcLoggable{
 
   val toolBox: ToolBox[universe.type] = runtimeMirror(getClass.getClassLoader).mkToolBox()
   private val memoClassPool = new Memo[ClassLoader, ClassPool]
@@ -50,6 +51,7 @@ object DynamicUtil {
    * @return compiled Full[function|object|class] or Failure
    */
   def compileScalaCode[T](code: String): Box[T] = {
+    logger.debug(s"code.api.util.DynamicUtil.compileScalaCode.size is ${dynamicCompileResult.size()}")
     val compiledResult: Box[Any] = dynamicCompileResult.computeIfAbsent(code, _ => {
       val tree = try {
         toolBox.parse(code)

diff --git a/obp-api/src/main/scala/code/api/v1_4_0/JSONFactory1_4_0.scala b/obp-api/src/main/scala/code/api/v1_4_0/JSONFactory1_4_0.scala
@@ -1,6 +1,7 @@
 package code.api.v1_4_0
 
 import code.api.berlin.group.v1_3.JvalueCaseClass
+import code.api.cache.Caching
 import java.util.Date
 
 import code.api.util.APIUtil.{EmptyBody, PrimaryDataBody, ResourceDoc}
@@ -21,10 +22,12 @@ import net.liftweb.json.{Formats, JDouble, JInt, JString}
 import net.liftweb.json.JsonAST.{JArray, JBool, JNothing, JObject, JValue}
 import net.liftweb.util.StringHelpers
 import code.util.Helper.MdcLoggable
+import com.tesobe.CacheKeyFromArguments
 import org.apache.commons.lang3.StringUtils
-import java.util.concurrent.ConcurrentHashMap
 import java.util.regex.Pattern
 import java.lang.reflect.Field
+import java.util.UUID.randomUUID
+import scala.concurrent.duration._
 
 object JSONFactory1_4_0 extends MdcLoggable{
   implicit def formats: Formats = CustomJsonFormats.formats
@@ -515,79 +518,97 @@ object JSONFactory1_4_0 extends MdcLoggable{
     jsonFieldsDescription.mkString(jsonTitleType,"","\n")
   }
 
-  private val createResourceDocJsonMemo = new ConcurrentHashMap[ResourceDoc, ResourceDocJson]
+  val createResourceDocJsonTTL : Int = APIUtil.getPropsValue(s"createResourceDocJson.cache.ttl.seconds", "86400").toInt
+
+  def createResourceDocJsonCached(resourceDocUpdatedTags: ResourceDoc, 
+    isVersion4OrHigher:Boolean, locale: Option[String], 
+    urlParametersI18n:String ,
+    jsonRequestBodyFieldsI18n:String, 
+    jsonResponseBodyFieldsI18n:String
+  ): ResourceDocJson = {
+    /**
+     * Please note that "var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)"
+     * is just a temporary value field with UUID values in order to prevent any ambiguity.
+     * The real value will be assigned by Macro during compile time at this line of a code:
+     * https://github.com/OpenBankProject/scala-macros/blob/master/macros/src/main/scala/com/tesobe/CacheKeyFromArgumentsMacro.scala#L49
+     */
+    var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)
+    CacheKeyFromArguments.buildCacheKey {
+      Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(createResourceDocJsonTTL second) {
+        // There are multiple flavours of markdown. For instance, original markdown emphasises underscores (surrounds _ with (<em>))
+        // But we don't want to have to escape underscores (\_) in our documentation
+        // Thus we use a flavour of markdown that ignores underscores in words. (Github markdown does this too)
+        // We return html rather than markdown to the consumer so they don't have to bother with these questions.
+
+        //Here area some endpoints, which should not be added the description:
+        // 1st: Dynamic entity endpoint,
+        // 2rd: Dynamic endpoint endpoints,
+        // 3rd: all the user created endpoints,
+        val fieldsDescription =
+        if (resourceDocUpdatedTags.tags.toString.contains("Dynamic-Entity")
+          || resourceDocUpdatedTags.tags.toString.contains("Dynamic-Endpoint")
+          || resourceDocUpdatedTags.roles.toString.contains("DynamicEntity")
+          || resourceDocUpdatedTags.roles.toString.contains("DynamicEntities")
+          || resourceDocUpdatedTags.roles.toString.contains("DynamicEndpoint")) {
+          ""
+        } else {
+          //1st: prepare the description from URL
+          val urlParametersDescription: String = prepareUrlParameterDescription(resourceDocUpdatedTags.requestUrl, urlParametersI18n)
+          //2rd: get the fields description from the post json body:
+          val exampleRequestBodyFieldsDescription =
+            if (resourceDocUpdatedTags.requestVerb == "POST") {
+              prepareJsonFieldDescription(resourceDocUpdatedTags.exampleRequestBody, "request", jsonRequestBodyFieldsI18n, jsonResponseBodyFieldsI18n)
+            } else {
+              ""
+            }
+          //3rd: get the fields description from the response body:
+          //response body can be a nest class, need to loop all the fields.
+          val responseFieldsDescription = prepareJsonFieldDescription(resourceDocUpdatedTags.successResponseBody, "response", jsonRequestBodyFieldsI18n, jsonResponseBodyFieldsI18n)
+          urlParametersDescription ++ exampleRequestBodyFieldsDescription ++ responseFieldsDescription
+        }
 
+        val resourceDocDescription = I18NUtil.ResourceDocTranslation.translate(
+          I18NResourceDocField.DESCRIPTION,
+          resourceDocUpdatedTags.operationId,
+          locale,
+          resourceDocUpdatedTags.description.stripMargin.trim
+        )
+        val description = resourceDocDescription ++ fieldsDescription
+        val summary = resourceDocUpdatedTags.summary.replaceFirst("""\.(\s*)$""", "$1") // remove the ending dot in summary
+        val translatedSummary = I18NUtil.ResourceDocTranslation.translate(I18NResourceDocField.SUMMARY, resourceDocUpdatedTags.operationId, locale, summary)
+
+        ResourceDocJson(
+          operation_id = resourceDocUpdatedTags.operationId,
+          request_verb = resourceDocUpdatedTags.requestVerb,
+          request_url = resourceDocUpdatedTags.requestUrl,
+          summary = translatedSummary,
+          // Strip the margin character (|) and line breaks and convert from markdown to html
+          description = PegdownOptions.convertPegdownToHtmlTweaked(description), //.replaceAll("\n", ""),
+          description_markdown = description,
+          example_request_body = resourceDocUpdatedTags.exampleRequestBody,
+          success_response_body = resourceDocUpdatedTags.successResponseBody,
+          error_response_bodies = resourceDocUpdatedTags.errorResponseBodies,
+          implemented_by = ImplementedByJson(resourceDocUpdatedTags.implementedInApiVersion.fullyQualifiedVersion, resourceDocUpdatedTags.partialFunctionName), // was resourceDocUpdatedTags.implementedInApiVersion.noV
+          tags = resourceDocUpdatedTags.tags.map(i => i.tag),
+          typed_request_body = createTypedBody(resourceDocUpdatedTags.exampleRequestBody),
+          typed_success_response_body = createTypedBody(resourceDocUpdatedTags.successResponseBody),
+          roles = resourceDocUpdatedTags.roles,
+          is_featured = resourceDocUpdatedTags.isFeatured,
+          special_instructions = PegdownOptions.convertPegdownToHtmlTweaked(resourceDocUpdatedTags.specialInstructions.getOrElse("").stripMargin),
+          specified_url = resourceDocUpdatedTags.specifiedUrl.getOrElse(""),
+          connector_methods = resourceDocUpdatedTags.connectorMethods,
+          created_by_bank_id = if (isVersion4OrHigher) resourceDocUpdatedTags.createdByBankId else None // only for V400 we show the bankId
+        )
+      }
+    }
+  }
+
   def createResourceDocJson(rd: ResourceDoc, isVersion4OrHigher:Boolean, locale: Option[String], urlParametersI18n:String ,jsonRequestBodyFieldsI18n:String, jsonResponseBodyFieldsI18n:String) : ResourceDocJson = {
     // We MUST recompute all resource doc values due to translation via Web UI props
     val endpointTags = getAllEndpointTagsBox(rd.operationId).map(endpointTag =>ResourceDocTag(endpointTag.tagName))
-    val resourceDocUpdatedTags: ResourceDoc = rd.copy(tags = endpointTags++ rd.tags)
-    logger.debug(s"createResourceDocJson createResourceDocJsonMemo.size is ${createResourceDocJsonMemo.size()}")
-    createResourceDocJsonMemo.compute(resourceDocUpdatedTags, (k, v) => {
-      // There are multiple flavours of markdown. For instance, original markdown emphasises underscores (surrounds _ with (<em>))
-      // But we don't want to have to escape underscores (\_) in our documentation
-      // Thus we use a flavour of markdown that ignores underscores in words. (Github markdown does this too)
-      // We return html rather than markdown to the consumer so they don't have to bother with these questions.
-
-      //Here area some endpoints, which should not be added the description:
-      // 1st: Dynamic entity endpoint,
-      // 2rd: Dynamic endpoint endpoints,
-      // 3rd: all the user created endpoints,
-      val fieldsDescription =
-      if(resourceDocUpdatedTags.tags.toString.contains("Dynamic-Entity")
-        ||resourceDocUpdatedTags.tags.toString.contains("Dynamic-Endpoint")
-        ||resourceDocUpdatedTags.roles.toString.contains("DynamicEntity")
-        ||resourceDocUpdatedTags.roles.toString.contains("DynamicEntities")
-        ||resourceDocUpdatedTags.roles.toString.contains("DynamicEndpoint")) {
-        ""
-      } else{
-        //1st: prepare the description from URL
-        val urlParametersDescription: String = prepareUrlParameterDescription(resourceDocUpdatedTags.requestUrl, urlParametersI18n)
-        //2rd: get the fields description from the post json body:
-        val exampleRequestBodyFieldsDescription =
-          if (resourceDocUpdatedTags.requestVerb=="POST" ){
-            prepareJsonFieldDescription(resourceDocUpdatedTags.exampleRequestBody,"request", jsonRequestBodyFieldsI18n, jsonResponseBodyFieldsI18n)
-          } else {
-            ""
-          }
-        //3rd: get the fields description from the response body:
-        //response body can be a nest class, need to loop all the fields.
-        val responseFieldsDescription = prepareJsonFieldDescription(resourceDocUpdatedTags.successResponseBody,"response", jsonRequestBodyFieldsI18n, jsonResponseBodyFieldsI18n)
-        urlParametersDescription ++ exampleRequestBodyFieldsDescription ++ responseFieldsDescription
-      }
+    val updatedTagsResourceDoc: ResourceDoc = rd.copy(tags = endpointTags++ rd.tags)
 
-      val resourceDocDescription = I18NUtil.ResourceDocTranslation.translate(
-        I18NResourceDocField.DESCRIPTION,
-        resourceDocUpdatedTags.operationId, 
-        locale, 
-        resourceDocUpdatedTags.description.stripMargin.trim
-      )
-      val description = resourceDocDescription ++ fieldsDescription
-      val summary = resourceDocUpdatedTags.summary.replaceFirst("""\.(\s*)$""", "$1") // remove the ending dot in summary
-      val translatedSummary = I18NUtil.ResourceDocTranslation.translate(I18NResourceDocField.SUMMARY, resourceDocUpdatedTags.operationId, locale, summary)
-
-      ResourceDocJson(
-        operation_id = resourceDocUpdatedTags.operationId,
-        request_verb = resourceDocUpdatedTags.requestVerb,
-        request_url = resourceDocUpdatedTags.requestUrl,
-        summary = translatedSummary,
-        // Strip the margin character (|) and line breaks and convert from markdown to html
-        description = PegdownOptions.convertPegdownToHtmlTweaked(description), //.replaceAll("\n", ""),
-        description_markdown = description,
-        example_request_body = resourceDocUpdatedTags.exampleRequestBody,
-        success_response_body = resourceDocUpdatedTags.successResponseBody,
-        error_response_bodies = resourceDocUpdatedTags.errorResponseBodies,
-        implemented_by = ImplementedByJson(resourceDocUpdatedTags.implementedInApiVersion.fullyQualifiedVersion, resourceDocUpdatedTags.partialFunctionName), // was resourceDocUpdatedTags.implementedInApiVersion.noV
-        tags = resourceDocUpdatedTags.tags.map(i => i.tag),
-        typed_request_body = createTypedBody(resourceDocUpdatedTags.exampleRequestBody),
-        typed_success_response_body = createTypedBody(resourceDocUpdatedTags.successResponseBody),
-        roles = resourceDocUpdatedTags.roles,
-        is_featured = resourceDocUpdatedTags.isFeatured,
-        special_instructions = PegdownOptions.convertPegdownToHtmlTweaked(resourceDocUpdatedTags.specialInstructions.getOrElse("").stripMargin),
-        specified_url = resourceDocUpdatedTags.specifiedUrl.getOrElse(""),
-        connector_methods = resourceDocUpdatedTags.connectorMethods,
-        created_by_bank_id= if (isVersion4OrHigher) rd.createdByBankId else None // only for V400 we show the bankId
-      )
-    }) 
+    createResourceDocJsonCached(updatedTagsResourceDoc, isVersion4OrHigher:Boolean, locale: Option[String], urlParametersI18n:String ,jsonRequestBodyFieldsI18n:String, jsonResponseBodyFieldsI18n:String)
   }
 
   def createResourceDocsJson(resourceDocList: List[ResourceDoc], isVersion4OrHigher:Boolean, locale: Option[String]) : ResourceDocsJson = {