From 8d9396c29380fe858d89fbf2bfa7399e73715fdb Mon Sep 17 00:00:00 2001 From: xlinliu Date: Wed, 10 Jan 2024 10:17:50 +0800 Subject: [PATCH 1/7] fix concurrent getAppConnManager bug --- .../dss/appconn/manager/impl/AbstractAppConnManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java index b13b8b6f09..bd47037e31 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java @@ -59,7 +59,7 @@ public abstract class AbstractAppConnManager implements AppConnManager { private AppConnResourceService appConnResourceService; private AppConnRefreshThread appConnRefreshThread; - private static AppConnManager appConnManager; + private static volatile AppConnManager appConnManager; private static boolean lazyLoad = false; public static void setLazyLoad() { @@ -87,8 +87,8 @@ public static AppConnManager getAppConnManager() { LOGGER.info("The instance of AppConnManager is {}.", appConnManager.getClass().getName()); appConnManager.init(); } + return appConnManager; } - return appConnManager; } @Override From 685541dec2798bb02c6d7daf19e70587cb84ffaf Mon Sep 17 00:00:00 2001 From: xlinliu Date: Wed, 10 Jan 2024 11:14:14 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=8F=90=E4=BA=A4?= =?UTF-8?q?=E5=B7=A5=E4=BD=9C=E6=B5=81=E8=BF=90=E8=A1=8C=E7=9A=84linkis=20?= =?UTF-8?q?token?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conf/dss-framework-orchestrator-server.properties | 1 - .../execution/conf/LinkisJobExecutionConfiguration.java | 2 +- .../node/execution/service/impl/BuildJobActionImpl.java | 7 +++++-- .../linkis-jobtype/src/main/resources/plugin.properties | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/conf/dss-framework-orchestrator-server.properties b/conf/dss-framework-orchestrator-server.properties index 52175a2608..b70b14792b 100644 --- a/conf/dss-framework-orchestrator-server.properties +++ b/conf/dss-framework-orchestrator-server.properties @@ -60,7 +60,6 @@ wds.linkis.server.version=v1 wds.linkis.server.socket.mode=true wds.linkis.client.flow.adminuser=ws -wds.linkis.client.flow.author.user.token=WS-AUTH wds.linkis.server.component.exclude.classes=org.apache.linkis.entranceclient.conf.ClientForEntranceSpringConfiguration,org.apache.linkis.entranceclient.conf.ClientSpringConfiguration diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java index d8b1254ed2..f057f3a8f4 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java @@ -61,7 +61,7 @@ public class LinkisJobExecutionConfiguration { public final static CommonVars LINKIS_ADMIN_USER = CommonVars.apply("wds.linkis.client.flow.adminuser","ws"); - public final static CommonVars LINKIS_AUTHOR_USER_TOKEN = CommonVars.apply("wds.linkis.client.flow.author.user.token","WS-AUTH"); + public final static CommonVars LINKIS_AUTHOR_USER_TOKEN = CommonVars.apply("wds.linkis.client.flow.author.user.token","admin-kmsnd"); public final static CommonVars LINKIS_JOB_CREATOR = CommonVars.apply("wds.linkis.flow.job.creator","nodeexecution"); diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java index 95f82b82db..bb6845acb3 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java @@ -148,15 +148,18 @@ public JobSubmitAction getSubmitAction(Job job) throws LinkisJobExecutionErrorEx JobSubmitAction.Builder builder = JobSubmitAction.builder() .addExecuteCode(code) - .setUser(job.getUser()) .addExecuteUser(job.getUser()) .setParams(paramMapCopy) .setLabels(labels) .setRuntimeParams(job.getRuntimeParams()); if (job instanceof LinkisJob) { + LinkisJob linkisJob = (LinkisJob) job; + builder = builder.setUser(linkisJob.getSubmitUser()); Map source = new HashMap<>(); - source.putAll(((LinkisJob) job).getSource()); + source.putAll(linkisJob.getSource()); builder = builder.setSource(source); + }else{ + builder = builder.setUser(job.getUser()); } // 将execute接口带来的额外variable参数，带进来 todo check Map propMap = new HashMap<>(); diff --git a/plugins/azkaban/linkis-jobtype/src/main/resources/plugin.properties b/plugins/azkaban/linkis-jobtype/src/main/resources/plugin.properties index e5d15ce7bd..182163d532 100644 --- a/plugins/azkaban/linkis-jobtype/src/main/resources/plugin.properties +++ b/plugins/azkaban/linkis-jobtype/src/main/resources/plugin.properties @@ -16,6 +16,6 @@ wds.linkis.gateway.url.v1=http://127.0.0.1:9001 wds.linkis.gateway.url.v0=http://127.0.0.1:9001 -wds.linkis.client.flow.author.user.token=WS-AUTH +wds.linkis.client.flow.author.user.token=admin-kmsnd wds.linkis.flow.job.creator=scheduler wds.linkis.flow.job.creator.v1=schedulis From fb6fd4d4a91be3ca441c756cf639958519f348ab Mon Sep 17 00:00:00 2001 From: xlinliu Date: Wed, 10 Jan 2024 15:42:10 +0800 Subject: [PATCH 3/7] appconn refresh fault tolerance --- .../dss/appconn/manager/impl/AbstractAppConnManager.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java index bd47037e31..14f51e41e5 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java @@ -53,7 +53,7 @@ public abstract class AbstractAppConnManager implements AppConnManager { private final AppConnLoader appConnLoader = AppConnLoaderFactory.getAppConnLoader(); private final Map appConns = new HashMap<>(); - private boolean isLoaded = false; + private volatile boolean isLoaded = false; private List appConnList = null; AppConnInfoService appConnInfoService; private AppConnResourceService appConnResourceService; @@ -223,10 +223,11 @@ private void lazyLoadAppConns() { if(lazyLoad){ LOGGER.info("lazyLoad set to true,isLoaded={}",isLoaded); } - if (lazyLoad && !isLoaded) { + if ( !isLoaded) { synchronized (this.appConns) { - if (lazyLoad && !isLoaded) { + if ( !isLoaded) { loadAppConns(); + isLoaded=true; } } } From c9c89902b34ccd66eac0a46d41d7bd39a85f4cd1 Mon Sep 17 00:00:00 2001 From: yuankang134 <373346037@qq.com> Date: Thu, 11 Jan 2024 09:40:32 +0800 Subject: [PATCH 4/7] convert symbol "--" to "\-\-" --- .../impl/ApiServiceQueryServiceImpl.java | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java index fbfb1f8033..0bf92e1da5 100644 --- a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java @@ -23,10 +23,8 @@ import com.webank.wedatasphere.dss.apiservice.core.bo.LinkisExecuteResult; import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration; import com.webank.wedatasphere.dss.apiservice.core.constant.ParamType; -import com.webank.wedatasphere.dss.apiservice.core.constant.ParamTypeEnum; import com.webank.wedatasphere.dss.apiservice.core.constant.RequireEnum; import com.webank.wedatasphere.dss.apiservice.core.dao.*; -import com.webank.wedatasphere.dss.apiservice.core.exception.ApiExecuteException; import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceQueryException; import com.webank.wedatasphere.dss.apiservice.core.execute.ApiServiceExecuteJob; import com.webank.wedatasphere.dss.apiservice.core.execute.DefaultApiServiceJob; @@ -72,6 +70,8 @@ import java.sql.SQLException; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import static java.util.stream.Collectors.toMap; @@ -80,7 +80,8 @@ @Service public class ApiServiceQueryServiceImpl implements ApiServiceQueryService { private static final Logger LOG = LoggerFactory.getLogger(ApiServiceQueryServiceImpl.class); - + private static final Pattern pattern = Pattern.compile("--+"); + private static final String REPLACEMENT = "\\-"; Map runJobs = new HashMap<>(); @@ -215,9 +216,14 @@ public LinkisExecuteResult query(String path, } // 用户请求的参数值注入检查，排除token - for(String k: reqParams.keySet()){ + for(Map.Entry entry: reqParams.entrySet()){ + String k = entry.getKey(); + String v = String.valueOf(entry.getValue()); + if (v.contains("--")){ + entry.setValue(replaceSymbol(v)); + } if(!k.equals(ApiServiceConfiguration.API_SERVICE_TOKEN_KEY.getValue()) - && SQLCheckUtil.doParamInjectionCheck(reqParams.get(k).toString())) { + && SQLCheckUtil.doParamInjectionCheck((String) reqParams.get(k))) { // 如果注入直接返回null LOG.warn("用户参数存在非法的关键字{}", reqParams.get(k).toString()); return null; @@ -543,4 +549,21 @@ private static String getRunTypeFromScriptsPath(String scriptsPath) { return res; } + + private static String replaceSymbol(String str) { + StringBuffer sb = new StringBuffer(); + Matcher matcher = pattern.matcher(str); + while (matcher.find()){ + String match = matcher.group(); + int length = match.length(); + StringBuilder replacement = new StringBuilder(); + for (int i = 0; i < length; i++) { + replacement.append(REPLACEMENT); + } + //避免将replacement识别为正则，将替换字符追加到sb中 + matcher.appendReplacement(sb, Matcher.quoteReplacement(replacement.toString())); + } + matcher.appendTail(sb); + return sb.toString(); + } } From fff021c58642de8f9b4a9a7e08a1d71f00270d96 Mon Sep 17 00:00:00 2001 From: xlinliu Date: Thu, 11 Jan 2024 21:37:33 +0800 Subject: [PATCH 5/7] Revert "appconn refresh fault tolerance" This reverts commit fb6fd4d4a91be3ca441c756cf639958519f348ab. --- .../dss/appconn/manager/impl/AbstractAppConnManager.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java index 14f51e41e5..bd47037e31 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java @@ -53,7 +53,7 @@ public abstract class AbstractAppConnManager implements AppConnManager { private final AppConnLoader appConnLoader = AppConnLoaderFactory.getAppConnLoader(); private final Map appConns = new HashMap<>(); - private volatile boolean isLoaded = false; + private boolean isLoaded = false; private List appConnList = null; AppConnInfoService appConnInfoService; private AppConnResourceService appConnResourceService; @@ -223,11 +223,10 @@ private void lazyLoadAppConns() { if(lazyLoad){ LOGGER.info("lazyLoad set to true,isLoaded={}",isLoaded); } - if ( !isLoaded) { + if (lazyLoad && !isLoaded) { synchronized (this.appConns) { - if ( !isLoaded) { + if (lazyLoad && !isLoaded) { loadAppConns(); - isLoaded=true; } } } From e8a88291d8f77156a0a23d6cb616532fe91a7a64 Mon Sep 17 00:00:00 2001 From: xlinliu Date: Thu, 11 Jan 2024 21:38:13 +0800 Subject: [PATCH 6/7] appconn refresh fault tolerance --- .../dss/appconn/manager/impl/AbstractAppConnManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java index bd47037e31..a6d019e7ab 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java @@ -53,7 +53,7 @@ public abstract class AbstractAppConnManager implements AppConnManager { private final AppConnLoader appConnLoader = AppConnLoaderFactory.getAppConnLoader(); private final Map appConns = new HashMap<>(); - private boolean isLoaded = false; + private volatile boolean isLoaded = false; private List appConnList = null; AppConnInfoService appConnInfoService; private AppConnResourceService appConnResourceService; From 0df9b19e1dcc649a82e73e6fca197d5382bff4f0 Mon Sep 17 00:00:00 2001 From: yuankang134 <373346037@qq.com> Date: Mon, 15 Jan 2024 11:28:15 +0800 Subject: [PATCH 7/7] add logs --- .../project/service/impl/DSSProjectServiceImpl.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectServiceImpl.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectServiceImpl.java index 17187a6117..804712fb83 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectServiceImpl.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectServiceImpl.java @@ -192,6 +192,9 @@ public List getListByParam(ProjectQueryRequest projectRequest) String editPriv = projectVo.getId() + KEY_SPLIT + ProjectUserPrivEnum.PRIV_EDIT.getRank() + KEY_SPLIT + projectRequest.getUsername(); + LOGGER.info("user:{} get project privilege info ,workspaceId:{}, projectId:{}, projectName:{}, pusername:{}, editPriv:{}", + projectRequest.getUsername(), projectRequest.getWorkspaceId(), projectVo.getId(), projectVo.getName(), pusername, editPriv); + Map> userPricMap = new HashMap<>(); String[] tempstrArr = pusername.split(MODE_SPLIT); @@ -212,6 +215,9 @@ public List getListByParam(ProjectQueryRequest projectRequest) projectResponse.setEditUsers(CollectionUtils.isEmpty(editUsers) ? new ArrayList<>() : editUsers.stream().distinct().collect(Collectors.toList())); projectResponse.setReleaseUsers(CollectionUtils.isEmpty(releaseUsers) ? new ArrayList<>() : releaseUsers.stream().distinct().collect(Collectors.toList())); + LOGGER.info("user:{} get project access users info, workspaceId:{}, projectId:{}, projectName:{}, accessUsers:{}, editUsers:{}, releaseUsers:{}", + projectRequest.getUsername(), projectRequest.getWorkspaceId(), projectVo.getId(), projectVo.getName(), accessUsers, editUsers, releaseUsers); + // 用户是否具有编辑权限 编辑权限和创建者都有 if (!StringUtils.isEmpty(pusername) && (pusername.contains(editPriv) ||