更新发布工作流时节点内容添加关键字检查

dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/DSSFrameworkOrchestratorRestful.java

@@ -763,4 +763,14 @@ public Message getAllOrchestratorName(HttpServletRequest httpServletRequest,
         return Message.ok().data("data", orchestratorService.getAllOrchestratorName(workspaceId,projectName));
     }
 
+
+    @RequestMapping(value = "publishFlowCheck",method = RequestMethod.GET)
+    public Message publishFlowCheck(@RequestParam("orchestratorId") Long orchestratorId,
+                                    @RequestParam("projectId") Long projectId) throws  DSSErrorException{
+
+        Workspace workspace = SSOHelper.getWorkspace(httpServletRequest);
+
+        return  Message.ok().data("data",orchestratorPluginService.getNotContainsKeywordsNode(orchestratorId,projectId,workspace));
+    }
+
 }

dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorPluginService.java

@@ -58,4 +58,6 @@ public interface OrchestratorPluginService {
     String diffStatus(Long taskId) throws DSSErrorException;
 
     OrchestratorDiffDirVo diffContent(Long taskId);
+
+    Map<String,List<String>> getNotContainsKeywordsNode(long orchestratorId,long projectId,Workspace workspace) throws  DSSErrorException;
 }

dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorPluginServiceImpl.java

@@ -18,6 +18,7 @@
 
 import com.google.common.reflect.TypeToken;
 import com.google.gson.Gson;
+import com.google.gson.JsonElement;
 import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum;
 import com.webank.wedatasphere.dss.common.entity.BmlResource;
 import com.webank.wedatasphere.dss.common.entity.project.DSSProject;
@@ -32,6 +33,8 @@
 import com.webank.wedatasphere.dss.common.protocol.project.ProjectUserAuthResponse;
 import com.webank.wedatasphere.dss.common.service.BMLService;
 import com.webank.wedatasphere.dss.common.utils.*;
+import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspace;
+import com.webank.wedatasphere.dss.framework.workspace.dao.DSSWorkspaceMapper;
 import com.webank.wedatasphere.dss.git.common.protocol.GitTree;
 import com.webank.wedatasphere.dss.git.common.protocol.request.*;
 import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse;
@@ -79,6 +82,7 @@
 import java.nio.file.Paths;
 import java.util.*;
 import java.util.concurrent.ExecutorService;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 
@@ -110,6 +114,9 @@ public class OrchestratorPluginServiceImpl implements OrchestratorPluginService
     @Autowired
     private OrchestratorPluginService orchestratorPluginService;
 
+    @Autowired
+    private DSSWorkspaceMapper dssWorkspaceMapper;
+
     @Autowired
     @Qualifier("workflowBmlService")
     private BMLService bmlService;
@@ -386,9 +393,150 @@ private GitCommitResponse submitWorkflowToBML(Long taskId, OrchestratorSubmitReq
         lockMapper.updateOrchestratorStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_PUSH);
         // 更新commitId
         lockMapper.updateOrchestratorVersionCommitId(commit.getCommitId(), flowId);
+
+        updateOrchestratorNotContainsKeywordsNode(flowRequest,orchestrator,username,workspace,commit.getCommitId());
+
         return commit;
     }
 
+    @Override
+    public Map<String,List<String>> getNotContainsKeywordsNode(long orchestratorId,long projectId,Workspace workspace) throws DSSErrorException {
+
+        DSSWorkspace dssWorkspace = dssWorkspaceMapper.getWorkspace((int) workspace.getWorkspaceId());
+        if(disabledKeywordsCheck(dssWorkspace)){
+            throw  new DSSErrorException(90058,"当前工作空间未启用工作流关键字校验");
+        }
+
+        ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest();
+        projectInfoRequest.setProjectId(projectId);
+
+        DSSProject project = RpcAskUtils.processAskException(DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender()
+                .ask(projectInfoRequest), DSSProject.class, ProjectInfoRequest.class);
+
+        // 项目未接入git
+        if(project.getAssociateGit() == null || !project.getAssociateGit()){
+            throw  new DSSErrorException(90058,"当前工作流所属项目未接入git");
+        }
+
+        String json = orchestratorMapper.getOrchestratorNotContainsKeywordsNode(orchestratorId);
+
+        if(StringUtils.isNotEmpty(json)){
+            return DSSCommonUtils.COMMON_GSON.fromJson(json,new TypeToken<List<Map<String, List<String>>>>() {
+            }.getType());
+        }
+
+        return null;
+
+
+    }
+
+    private boolean disabledKeywordsCheck(DSSWorkspace dssWorkspace){
+
+        // 命名空间 禁用关键字检查
+        return dssWorkspace == null || StringUtils.isEmpty(dssWorkspace.getEnabledFlowKeywordsCheck())
+                || !dssWorkspace.getEnabledFlowKeywordsCheck().equals("1");
+    }
+
+
+    private void updateOrchestratorNotContainsKeywordsNode(OrchestratorSubmitRequest flowRequest, DSSOrchestratorInfo orchestrator,
+                                                           String username, Workspace workspace,String commitId){
+
+        DSSWorkspace dssWorkspace = dssWorkspaceMapper.getWorkspace((int) workspace.getWorkspaceId());
+
+        if(disabledKeywordsCheck(dssWorkspace)){
+            return;
+        }
+
+        GitDiffResponse gitDiffResponse = diffPublish(orchestrator.getName(), commitId,username,
+                workspace.getWorkspaceId(), flowRequest.getProjectName());
+
+        if (gitDiffResponse == null) {
+            LOGGER.info("gitDiffResponse change is empty");
+            return;
+        }
+
+        List<String> paths = new ArrayList<>();
+
+        gitDiffResponse.getCodeTree().forEach(gitTree -> {
+            getCodePath(paths,gitTree);
+        });
+
+        // 获取内容并检查
+        List<String> nodePathList = getNotContainsKeywordsNodePath(paths,commitId,workspace,flowRequest.getProjectName(),username);
+        Map<String,List<String>> map = new HashMap<>();
+        map.put(orchestrator.getName(),nodePathList);
+        String notContainsKeywordsNode =  DSSCommonUtils.COMMON_GSON.toJson(map);
+
+        orchestratorMapper.updateOrchestratorNotContainsKeywordsNode(flowRequest.getOrchestratorId(),notContainsKeywordsNode);
+    }
+
+
+    private void getCodePath(List<String> paths,GitTree gitTree){
+
+        if(MapUtils.isEmpty(gitTree.getChildren())){
+            return;
+        }
+
+        for (Map.Entry<String,GitTree> entry : gitTree.getChildren().entrySet()){
+            // 过滤后缀
+            if(StringUtils.endsWithAny(entry.getValue().getAbsolutePath(),".sql",".hql",".py")){
+                continue;
+            }
+
+            paths.add(gitTree.getAbsolutePath());
+
+            getCodePath(paths, entry.getValue());
+        }
+
+    }
+
+
+    private List<String> getNotContainsKeywordsNodePath(List<String> codePaths,String commitId,Workspace workspace,String projectName,String username){
+
+        List<String> nodePathList = new ArrayList<>();
+
+        if(CollectionUtils.isEmpty(codePaths)){
+            return  nodePathList;
+        }
+
+        GitDiffFileContentRequest diffFileContentRequest = new GitDiffFileContentRequest();
+        diffFileContentRequest.setFilePaths(codePaths);
+        diffFileContentRequest.setPublish(true);
+        diffFileContentRequest.setCommitId(commitId);
+        diffFileContentRequest.setUsername(username);
+        diffFileContentRequest.setWorkspaceId(workspace.getWorkspaceId());
+        diffFileContentRequest.setProjectName(projectName);
+        Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender();
+        // 批量获取脚本内容新
+        GitDiffFileContentResponse gitDiffFileContentResponse = RpcAskUtils.processAskException(
+                gitSender.ask(diffFileContentRequest), GitDiffFileContentResponse.class, GitDiffFileContentRequest.class);
+
+        if(gitDiffFileContentResponse  == null){
+            LOGGER.info("get script content isEmpty");
+            return null;
+        }
+
+        Pattern pattern = Pattern.compile("(create\\s+table|insert)\\s*",Pattern.CASE_INSENSITIVE);
+
+        for (GitFileContentResponse gitFileContentResponse: gitDiffFileContentResponse.getGitFileContentResponseList()){
+
+            String  content = gitFileContentResponse.getBefore();
+
+            // 内容为NULL或者已匹配到关键字 跳过
+            if(content == null || pattern.matcher(content).find()){
+                continue;
+            }
+
+            // 去掉/xxx.sql,hql,py 脚本名称
+            String nodePath = gitFileContentResponse.getFilePath().substring(0,gitFileContentResponse.getFilePath().lastIndexOf("/"));
+            nodePathList.add(nodePath);
+        }
+
+        return  nodePathList;
+
+    }
+
+
     public GitCommitResponse batchSubmitWorkflowToBML(List<Long> taskIdList, List<OrchestratorRelationVo> relationVos, String username, Workspace workspace, String projectName, String label, Long projectId, String comment) throws Exception {
 
         List<Long> flowIdList = relationVos.stream().map(OrchestratorRelationVo::getFlowId).collect(Collectors.toList());

dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceMapper.java

@@ -37,6 +37,7 @@ public interface DSSWorkspaceMapper {
             @Result(property = "lastUpdateTime", column = "last_update_time"),
             @Result(property = "lastUpdateUser", column = "last_update_user"),
             @Result(property = "enabledFlowKeywordsCheck",column = "enabled_flow_keywords_check"),
+            @Result(property = "isDefaultReference",column = "is_default_reference")
     })
     DSSWorkspace getWorkspace(@Param("workspaceId") int workspaceId);
 

dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffFileContentRequest.java

@@ -0,0 +1,50 @@
+package com.webank.wedatasphere.dss.git.common.protocol.request;
+
+import java.util.List;
+
+public class GitDiffFileContentRequest extends GitBaseRequest {
+
+
+    private String commitId;
+    /**
+     * 需获取内容的文件相对路径
+     */
+    private List<String> filePaths;
+
+    private String username;
+
+    private Boolean publish;
+
+
+    public String getCommitId() {
+        return commitId;
+    }
+
+    public void setCommitId(String commitId) {
+        this.commitId = commitId;
+    }
+
+    public List<String> getFilePaths() {
+        return filePaths;
+    }
+
+    public void setFilePaths(List<String> filePaths) {
+        this.filePaths = filePaths;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public Boolean getPublish() {
+        return publish;
+    }
+
+    public void setPublish(Boolean publish) {
+        this.publish = publish;
+    }
+}

dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffFileContentResponse.java

@@ -0,0 +1,25 @@
+package com.webank.wedatasphere.dss.git.common.protocol.request;
+
+import com.webank.wedatasphere.dss.git.common.protocol.response.GitFileContentResponse;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class GitDiffFileContentResponse {
+
+
+    private List<GitFileContentResponse> gitFileContentResponseList;
+
+
+    public GitDiffFileContentResponse(List<GitFileContentResponse> gitFileContentResponseList) {
+        this.gitFileContentResponseList = gitFileContentResponseList;
+    }
+
+    public List<GitFileContentResponse> getGitFileContentResponseList() {
+        return gitFileContentResponseList;
+    }
+
+    public void setGitFileContentResponseList(List<GitFileContentResponse> gitFileContentResponseList) {
+        this.gitFileContentResponseList = gitFileContentResponseList;
+    }
+}

dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitFileContentResponse.java

@@ -14,6 +14,8 @@ public class GitFileContentResponse{
     // 反显CommitId --仅发布时diff需要
     private String AfterCommitId;
 
+    // 文件路径
+    private String filePath;
 
     public GitFileContentResponse() {
     }
@@ -27,6 +29,15 @@ public GitFileContentResponse(String before, String after, String beforeAnnotate
         AfterCommitId = afterCommitId;
     }
 
+    public GitFileContentResponse(String before, String after, String beforeAnnotate, String beforeCommitId, String afterAnnotate, String afterCommitId, String filePath) {
+        this.before = before;
+        this.after = after;
+        this.beforeAnnotate = beforeAnnotate;
+        this.beforeCommitId = beforeCommitId;
+        AfterAnnotate = afterAnnotate;
+        AfterCommitId = afterCommitId;
+        this.filePath = filePath;
+    }
 
     public String getBefore() {
         return before;
@@ -75,4 +86,12 @@ public String getAfterCommitId() {
     public void setAfterCommitId(String afterCommitId) {
         AfterCommitId = afterCommitId;
     }
+
+    public String getFilePath() {
+        return filePath;
+    }
+
+    public void setFilePath(String filePath) {
+        this.filePath = filePath;
+    }
 }

dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitWorkflowManagerService.java

@@ -38,4 +38,7 @@ public interface DSSGitWorkflowManagerService {
     GitCommitResponse batchCommit(GitBatchCommitRequest request) throws DSSErrorException;
 
     Repository getRepository(File repoDir, String projectName, Long workspaceId, String gitUser, String gitToken, String gitUrl) throws DSSErrorException;
+
+    GitDiffFileContentResponse getDiffFileContent(GitDiffFileContentRequest request) throws DSSErrorException;
+
 }