From 236747bd83b4ea9ffb3c8993dd052eaaa2db6f0d Mon Sep 17 00:00:00 2001 From: sonikashah Date: Mon, 30 Dec 2024 14:28:25 +0530 Subject: [PATCH 1/2] Backend support for domain hierarchy listing --- .../service/jdbi3/DomainRepository.java | 49 ++++ .../service/jdbi3/EntityRepository.java | 2 +- .../resources/domains/DomainResource.java | 30 +++ .../elasticsearch/ElasticSearchClient.java | 166 ++++++++++---- .../search/opensearch/OpenSearchClient.java | 163 ++++++++----- .../service/util/EntityHierarchyList.java | 13 ++ .../resources/domains/DomainResourceTest.java | 215 ++++++++++++++++++ 7 files changed, 537 insertions(+), 101 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/util/EntityHierarchyList.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java index 40943f8bc185..5a6f6dcca48e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java @@ -18,10 +18,15 @@ import static org.openmetadata.service.Entity.DOMAIN; import static org.openmetadata.service.Entity.FIELD_ASSETS; +import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.entity.data.EntityHierarchy; import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; @@ -30,8 +35,11 @@ import org.openmetadata.schema.type.api.BulkOperationResult; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.domains.DomainResource; +import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.ResultList; @Slf4j public class DomainRepository extends EntityRepository { @@ -140,6 +148,47 @@ public EntityInterface getParentEntity(Domain entity, String fields) { : null; } + public List buildHierarchy(String fieldsParam, int limit) { + fieldsParam = EntityUtil.addField(fieldsParam, Entity.FIELD_PARENT); + Fields fields = getFields(fieldsParam); + ResultList resultList = listAfter(null, fields, new ListFilter(null), limit, null); + List domains = resultList.getData(); + + /* + Maintaining hierarchy in terms of EntityHierarchy to get all other fields of Domain like style, + which would have been restricted if built using hierarchy of Domain, as Domain.getChildren() returns List + and EntityReference does not support additional properties + */ + List rootDomains = new ArrayList<>(); + + Map entityHierarchyMap = + domains.stream() + .collect( + Collectors.toMap( + Domain::getId, + domain -> { + EntityHierarchy entityHierarchy = + JsonUtils.readValue(JsonUtils.pojoToJson(domain), EntityHierarchy.class); + entityHierarchy.setChildren(new ArrayList<>()); + return entityHierarchy; + })); + + for (Domain domain : domains) { + EntityHierarchy entityHierarchy = entityHierarchyMap.get(domain.getId()); + + if (domain.getParent() != null) { + EntityHierarchy parentHierarchy = entityHierarchyMap.get(domain.getParent().getId()); + if (parentHierarchy != null) { + parentHierarchy.getChildren().add(entityHierarchy); + } + } else { + rootDomains.add(entityHierarchy); + } + } + + return rootDomains; + } + public class DomainUpdater extends EntityUpdater { public DomainUpdater(Domain original, Domain updated, Operation operation) { super(original, updated, operation); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index c3d23fb876f2..a8a23f2a9219 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -2620,7 +2620,7 @@ public boolean isDelete() { * version goes to v-1 and new version v0 replaces v1 for the entity. * * - * @see TableRepository.TableUpdater#entitySpecificUpdate() for example. + * @see TableRepository.TableUpdater #entitySpecificUpdate() for example. */ public class EntityUpdater { private static volatile long sessionTimeoutMillis = 10L * 60 * 1000; // 10 minutes diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java index f3b446ce4c98..c335f6833de6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java @@ -45,6 +45,7 @@ import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.domains.CreateDomain; +import org.openmetadata.schema.entity.data.EntityHierarchy; import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.type.EntityHistory; @@ -57,6 +58,7 @@ import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.util.EntityHierarchyList; import org.openmetadata.service.util.ResultList; @Slf4j @@ -433,4 +435,32 @@ public Response delete( String name) { return deleteByName(uriInfo, securityContext, name, true, true); } + + @GET + @Path("/hierarchy") + @Operation( + operationId = "listDomainsHierarchy", + summary = "List domains in hierarchical order", + description = "Get a list of Domains in hierarchical order.", + responses = { + @ApiResponse( + responseCode = "200", + description = "List of Domains in hierarchical order", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = EntityHierarchyList.class))) + }) + public ResultList listHierarchy( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Fields requested in the returned resource", + schema = @Schema(type = "string", example = FIELDS)) + @QueryParam("fields") + String fieldsParam, + @DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam) { + + return new EntityHierarchyList(repository.buildHierarchy(fieldsParam, limitParam)); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 5ffe54a23ad1..402ce6311efd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -127,6 +127,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.json.JsonObject; import javax.net.ssl.SSLContext; @@ -468,50 +469,10 @@ public Response search(SearchRequest request, SubjectContext subjectContext) thr .getIndexMapping(GLOSSARY_TERM) .getIndexName(clusterAlias))) { searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query())); - - if (request.isGetHierarchy()) { - QueryBuilder baseQuery = - QueryBuilders.boolQuery() - .should(searchSourceBuilder.query()) - .should(QueryBuilders.matchPhraseQuery("fullyQualifiedName", request.getQuery())) - .should(QueryBuilders.matchPhraseQuery("name", request.getQuery())) - .should(QueryBuilders.matchPhraseQuery("displayName", request.getQuery())) - .should( - QueryBuilders.matchPhraseQuery( - "glossary.fullyQualifiedName", request.getQuery())) - .should(QueryBuilders.matchPhraseQuery("glossary.displayName", request.getQuery())) - .must(QueryBuilders.matchQuery("status", "Approved")) - .minimumShouldMatch(1); - searchSourceBuilder.query(baseQuery); - - SearchResponse searchResponse = - client.search( - new es.org.elasticsearch.action.search.SearchRequest(request.getIndex()) - .source(searchSourceBuilder), - RequestOptions.DEFAULT); - - // Extract parent terms from aggregation - BoolQueryBuilder parentTermQueryBuilder = QueryBuilders.boolQuery(); - Terms parentTerms = searchResponse.getAggregations().get("fqnParts_agg"); - - // Build es query to get parent terms for the user input query , to build correct hierarchy - if (!parentTerms.getBuckets().isEmpty() && !request.getQuery().equals("*")) { - parentTerms.getBuckets().stream() - .map(Terms.Bucket::getKeyAsString) - .forEach( - parentTerm -> - parentTermQueryBuilder.should( - QueryBuilders.matchQuery("fullyQualifiedName", parentTerm))); - - searchSourceBuilder.query( - parentTermQueryBuilder - .minimumShouldMatch(1) - .must(QueryBuilders.matchQuery("status", "Approved"))); - } - searchSourceBuilder.sort(SortBuilders.fieldSort("fullyQualifiedName").order(SortOrder.ASC)); - } } + buildHierarchyQuery(request, searchSourceBuilder, client); + /* for performance reasons ElasticSearch doesn't provide accurate hits if we enable trackTotalHits parameter it will try to match every result, count and return hits however in most cases for search results an approximate value is good enough. @@ -579,15 +540,89 @@ public Response getDocByID(String indexName, String entityId) throws IOException return getResponse(NOT_FOUND, "Document not found."); } + private void buildHierarchyQuery( + SearchRequest request, SearchSourceBuilder searchSourceBuilder, RestHighLevelClient client) + throws IOException { + + if (!request.isGetHierarchy()) { + return; + } + + String indexName = request.getIndex(); + String glossaryTermIndex = + Entity.getSearchRepository().getIndexMapping(GLOSSARY_TERM).getIndexName(clusterAlias); + String domainIndex = + Entity.getSearchRepository().getIndexMapping(DOMAIN).getIndexName(clusterAlias); + + BoolQueryBuilder baseQuery = + QueryBuilders.boolQuery() + .should(searchSourceBuilder.query()) + .should(QueryBuilders.matchPhraseQuery("fullyQualifiedName", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("name", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("displayName", request.getQuery())); + + if (indexName.equalsIgnoreCase(glossaryTermIndex)) { + baseQuery + .should(QueryBuilders.matchPhraseQuery("glossary.fullyQualifiedName", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("glossary.displayName", request.getQuery())) + .must(QueryBuilders.matchQuery("status", "Approved")); + } else if (indexName.equalsIgnoreCase(domainIndex)) { + baseQuery + .should(QueryBuilders.matchPhraseQuery("parent.fullyQualifiedName", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("parent.displayName", request.getQuery())); + } + + baseQuery.minimumShouldMatch(1); + searchSourceBuilder.query(baseQuery); + + SearchResponse searchResponse = + client.search( + new es.org.elasticsearch.action.search.SearchRequest(request.getIndex()) + .source(searchSourceBuilder), + RequestOptions.DEFAULT); + + Terms parentTerms = searchResponse.getAggregations().get("fqnParts_agg"); + + // Build es query to get parent terms for the user input query , to build correct hierarchy + // In case of default search , no need to get parent terms they are already present in the + // response + if (parentTerms != null + && !parentTerms.getBuckets().isEmpty() + && !request.getQuery().equals("*")) { + BoolQueryBuilder parentTermQueryBuilder = QueryBuilders.boolQuery(); + + parentTerms.getBuckets().stream() + .map(Terms.Bucket::getKeyAsString) + .forEach( + parentTerm -> + parentTermQueryBuilder.should( + QueryBuilders.matchQuery("fullyQualifiedName", parentTerm))); + if (indexName.equalsIgnoreCase(glossaryTermIndex)) { + parentTermQueryBuilder + .minimumShouldMatch(1) + .must(QueryBuilders.matchQuery("status", "Approved")); + } else { + parentTermQueryBuilder.minimumShouldMatch(1); + } + searchSourceBuilder.query(parentTermQueryBuilder); + } + + searchSourceBuilder.sort(SortBuilders.fieldSort("fullyQualifiedName").order(SortOrder.ASC)); + } + public List buildSearchHierarchy(SearchRequest request, SearchResponse searchResponse) { List response = new ArrayList<>(); - if (request - .getIndex() - .equalsIgnoreCase( - Entity.getSearchRepository() - .getIndexMapping(GLOSSARY_TERM) - .getIndexName(clusterAlias))) { + + String indexName = request.getIndex(); + String glossaryTermIndex = + Entity.getSearchRepository().getIndexMapping(GLOSSARY_TERM).getIndexName(clusterAlias); + String domainIndex = + Entity.getSearchRepository().getIndexMapping(DOMAIN).getIndexName(clusterAlias); + + if (indexName.equalsIgnoreCase(glossaryTermIndex)) { response = buildGlossaryTermSearchHierarchy(searchResponse); + } else if (indexName.equalsIgnoreCase(domainIndex)) { + response = buildDomainSearchHierarchy(searchResponse); } return response; } @@ -643,6 +678,36 @@ public List buildGlossaryTermSearchHierarchy(SearchResponse sea return new ArrayList<>(rootTerms.values()); } + public List buildDomainSearchHierarchy(SearchResponse searchResponse) { + Map entityHierarchyMap = + Arrays.stream(searchResponse.getHits().getHits()) + .map(hit -> JsonUtils.readValue(hit.getSourceAsString(), EntityHierarchy.class)) + .collect( + Collectors.toMap( + EntityHierarchy::getFullyQualifiedName, + entity -> { + entity.setChildren(new ArrayList<>()); + return entity; + })); + + List rootDomains = new ArrayList<>(); + + entityHierarchyMap + .values() + .forEach( + entity -> { + String parentFqn = getParentFQN(entity.getFullyQualifiedName()); + EntityHierarchy parentEntity = entityHierarchyMap.get(parentFqn); + if (parentEntity != null) { + parentEntity.getChildren().add(entity); + } else { + rootDomains.add(entity); + } + }); + + return rootDomains; + } + @Override public SearchResultListMapper listWithOffset( String filter, @@ -1797,7 +1862,10 @@ private static SearchSourceBuilder buildDomainsSearch(String query, int from, in buildSearchQueryBuilder(query, DomainIndex.getFields()); FunctionScoreQueryBuilder queryBuilder = boostScore(queryStringBuilder); HighlightBuilder hb = buildHighlights(new ArrayList<>()); - return searchBuilder(queryBuilder, hb, from, size); + SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size); + searchSourceBuilder.aggregation( + AggregationBuilders.terms("fqnParts_agg").field("fqnParts").size(1000)); + return addAggregation(searchSourceBuilder); } private static SearchSourceBuilder buildCostAnalysisReportDataSearch( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 4b3346b493be..2bc2b1375039 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -47,6 +47,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.json.JsonObject; import javax.net.ssl.SSLContext; @@ -462,53 +463,10 @@ public Response search(SearchRequest request, SubjectContext subjectContext) thr .getIndexMapping(GLOSSARY_TERM) .getIndexName(clusterAlias))) { searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query())); - - if (request.isGetHierarchy()) { - /* - Search for user input terms in name, fullyQualifiedName, displayName and glossary.fullyQualifiedName, glossary.displayName - */ - QueryBuilder baseQuery = - QueryBuilders.boolQuery() - .should(searchSourceBuilder.query()) - .should(QueryBuilders.matchPhraseQuery("fullyQualifiedName", request.getQuery())) - .should(QueryBuilders.matchPhraseQuery("name", request.getQuery())) - .should(QueryBuilders.matchPhraseQuery("displayName", request.getQuery())) - .should( - QueryBuilders.matchPhraseQuery( - "glossary.fullyQualifiedName", request.getQuery())) - .should(QueryBuilders.matchPhraseQuery("glossary.displayName", request.getQuery())) - .must(QueryBuilders.matchQuery("status", "Approved")) - .minimumShouldMatch(1); - searchSourceBuilder.query(baseQuery); - - SearchResponse searchResponse = - client.search( - new os.org.opensearch.action.search.SearchRequest(request.getIndex()) - .source(searchSourceBuilder), - RequestOptions.DEFAULT); - - // Extract parent terms from aggregation - BoolQueryBuilder parentTermQueryBuilder = QueryBuilders.boolQuery(); - Terms parentTerms = searchResponse.getAggregations().get("fqnParts_agg"); - - // Build es query to get parent terms for the user input query , to build correct hierarchy - if (!parentTerms.getBuckets().isEmpty() && !request.getQuery().equals("*")) { - parentTerms.getBuckets().stream() - .map(Terms.Bucket::getKeyAsString) - .forEach( - parentTerm -> - parentTermQueryBuilder.should( - QueryBuilders.matchQuery("fullyQualifiedName", parentTerm))); - - searchSourceBuilder.query( - parentTermQueryBuilder - .minimumShouldMatch(1) - .must(QueryBuilders.matchQuery("status", "Approved"))); - } - searchSourceBuilder.sort(SortBuilders.fieldSort("fullyQualifiedName").order(SortOrder.ASC)); - } } + buildHierarchyQuery(request, searchSourceBuilder, client); + /* for performance reasons OpenSearch doesn't provide accurate hits if we enable trackTotalHits parameter it will try to match every result, count and return hits however in most cases for search results an approximate value is good enough. @@ -571,15 +529,88 @@ public Response getDocByID(String indexName, String entityId) throws IOException return getResponse(NOT_FOUND, "Document not found."); } + private void buildHierarchyQuery( + SearchRequest request, SearchSourceBuilder searchSourceBuilder, RestHighLevelClient client) + throws IOException { + + if (!request.isGetHierarchy()) { + return; + } + + String indexName = request.getIndex(); + String glossaryTermIndex = + Entity.getSearchRepository().getIndexMapping(GLOSSARY_TERM).getIndexName(clusterAlias); + String domainIndex = + Entity.getSearchRepository().getIndexMapping(DOMAIN).getIndexName(clusterAlias); + + BoolQueryBuilder baseQuery = + QueryBuilders.boolQuery() + .should(searchSourceBuilder.query()) + .should(QueryBuilders.matchPhraseQuery("fullyQualifiedName", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("name", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("displayName", request.getQuery())); + + if (indexName.equalsIgnoreCase(glossaryTermIndex)) { + baseQuery + .should(QueryBuilders.matchPhraseQuery("glossary.fullyQualifiedName", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("glossary.displayName", request.getQuery())) + .must(QueryBuilders.matchQuery("status", "Approved")); + } else if (indexName.equalsIgnoreCase(domainIndex)) { + baseQuery + .should(QueryBuilders.matchPhraseQuery("parent.fullyQualifiedName", request.getQuery())) + .should(QueryBuilders.matchPhraseQuery("parent.displayName", request.getQuery())); + } + + baseQuery.minimumShouldMatch(1); + searchSourceBuilder.query(baseQuery); + + SearchResponse searchResponse = + client.search( + new os.org.opensearch.action.search.SearchRequest(request.getIndex()) + .source(searchSourceBuilder), + RequestOptions.DEFAULT); + + Terms parentTerms = searchResponse.getAggregations().get("fqnParts_agg"); + + // Build es query to get parent terms for the user input query , to build correct hierarchy + // In case of default search , no need to get parent terms they are already present in the + // response + if (parentTerms != null + && !parentTerms.getBuckets().isEmpty() + && !request.getQuery().equals("*")) { + BoolQueryBuilder parentTermQueryBuilder = QueryBuilders.boolQuery(); + + parentTerms.getBuckets().stream() + .map(Terms.Bucket::getKeyAsString) + .forEach( + parentTerm -> + parentTermQueryBuilder.should( + QueryBuilders.matchQuery("fullyQualifiedName", parentTerm))); + if (indexName.equalsIgnoreCase(glossaryTermIndex)) { + parentTermQueryBuilder + .minimumShouldMatch(1) + .must(QueryBuilders.matchQuery("status", "Approved")); + } else { + parentTermQueryBuilder.minimumShouldMatch(1); + } + searchSourceBuilder.query(parentTermQueryBuilder); + } + + searchSourceBuilder.sort(SortBuilders.fieldSort("fullyQualifiedName").order(SortOrder.ASC)); + } + public List buildSearchHierarchy(SearchRequest request, SearchResponse searchResponse) { List response = new ArrayList<>(); - if (request - .getIndex() - .equalsIgnoreCase( - Entity.getSearchRepository() - .getIndexMapping(GLOSSARY_TERM) - .getIndexName(clusterAlias))) { + String indexName = request.getIndex(); + String glossaryTermIndex = + Entity.getSearchRepository().getIndexMapping(GLOSSARY_TERM).getIndexName(clusterAlias); + String domainIndex = + Entity.getSearchRepository().getIndexMapping(DOMAIN).getIndexName(clusterAlias); + + if (indexName.equalsIgnoreCase(glossaryTermIndex)) { response = buildGlossaryTermSearchHierarchy(searchResponse); + } else if (indexName.equalsIgnoreCase(domainIndex)) { + response = buildDomainSearchHierarchy(searchResponse); } return response; } @@ -635,6 +666,36 @@ public List buildGlossaryTermSearchHierarchy(SearchResponse sea return new ArrayList<>(rootTerms.values()); } + public List buildDomainSearchHierarchy(SearchResponse searchResponse) { + Map entityHierarchyMap = + Arrays.stream(searchResponse.getHits().getHits()) + .map(hit -> JsonUtils.readValue(hit.getSourceAsString(), EntityHierarchy.class)) + .collect( + Collectors.toMap( + EntityHierarchy::getFullyQualifiedName, + entity -> { + entity.setChildren(new ArrayList<>()); + return entity; + })); + + List rootDomains = new ArrayList<>(); + + entityHierarchyMap + .values() + .forEach( + entity -> { + String parentFqn = getParentFQN(entity.getFullyQualifiedName()); + EntityHierarchy parentEntity = entityHierarchyMap.get(parentFqn); + if (parentEntity != null) { + parentEntity.getChildren().add(entity); + } else { + rootDomains.add(entity); + } + }); + + return rootDomains; + } + @Override public SearchResultListMapper listWithOffset( String filter, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityHierarchyList.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityHierarchyList.java new file mode 100644 index 000000000000..fe58dc390de3 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityHierarchyList.java @@ -0,0 +1,13 @@ +package org.openmetadata.service.util; + +import java.util.List; +import org.openmetadata.schema.entity.data.EntityHierarchy; + +public class EntityHierarchyList extends ResultList { + @SuppressWarnings("unused") + public EntityHierarchyList() {} + + public EntityHierarchyList(List data) { + super(data, null, null, data.size()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java index 7054353f2f6c..d793b5a15532 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DomainResourceTest.java @@ -3,6 +3,8 @@ import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.service.Entity.TABLE; import static org.openmetadata.service.security.SecurityUtil.authHeaders; @@ -17,15 +19,18 @@ import static org.openmetadata.service.util.TestUtils.assertResponse; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response.Status; import org.apache.http.client.HttpResponseException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.openmetadata.schema.api.domains.CreateDomain; import org.openmetadata.schema.api.domains.CreateDomain.DomainType; +import org.openmetadata.schema.entity.data.EntityHierarchy; import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.entity.type.Style; import org.openmetadata.schema.type.ChangeDescription; @@ -35,7 +40,9 @@ import org.openmetadata.service.jdbi3.TableRepository; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.domains.DomainResource.DomainList; +import org.openmetadata.service.util.EntityHierarchyList; import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.TestUtils; public class DomainResourceTest extends EntityResourceTest { public DomainResourceTest() { @@ -143,6 +150,214 @@ void patchWrongExperts(TestInfo test) throws IOException { String.format("user instance for %s not found", expertReference.getId())); } + @Test + void test_buildDomainNestedHierarchyFromSearch() throws HttpResponseException { + CreateDomain createRootDomain = + createRequest("rootDomain") + .withDisplayName("Global Headquarters") + .withDescription("Root Domain") + .withDomainType(DomainType.AGGREGATE) + .withStyle(null) + .withExperts(null); + Domain rootDomain = createEntity(createRootDomain, ADMIN_AUTH_HEADERS); + + CreateDomain createSecondLevelDomain = + createRequest("secondLevelDomain") + .withDisplayName("Operations Hub") + .withDescription("Second Level Domain") + .withDomainType(DomainType.AGGREGATE) + .withStyle(null) + .withExperts(null) + .withParent(rootDomain.getFullyQualifiedName()); + Domain secondLevelDomain = createEntity(createSecondLevelDomain, ADMIN_AUTH_HEADERS); + + CreateDomain createThirdLevelDomain = + createRequest("thirdLevelDomain") + .withDisplayName("Innovation Center") + .withDescription("Third Level Domain") + .withDomainType(DomainType.AGGREGATE) + .withStyle(null) + .withExperts(null) + .withParent(secondLevelDomain.getFullyQualifiedName()); + Domain thirdLevelDomain = createEntity(createThirdLevelDomain, ADMIN_AUTH_HEADERS); + + // Search for the displayName of third-level child domain and verify the hierarchy + String response = getResponseFormSearchWithHierarchy("domain_search_index", "*innovation*"); + List domains = JsonUtils.readObjects(response, EntityHierarchy.class); + + boolean isChild = + domains.stream() + .filter(domain -> "rootDomain".equals(domain.getName())) + .findFirst() + .map( + root -> + root.getChildren().stream() + .filter(domain -> "secondLevelDomain".equals(domain.getName())) + .flatMap(secondLevel -> secondLevel.getChildren().stream()) + .anyMatch(thirdLevel -> "thirdLevelDomain".equals(thirdLevel.getName()))) + .orElse(false); + + assertTrue( + isChild, + "thirdLevelDomain should be a child of secondLevelDomain, which should be a child of rootDomain"); + + // Search for the fqn of third-level child domain and verify the hierarchy + response = getResponseFormSearchWithHierarchy("domain_search_index", "*third*"); + domains = JsonUtils.readObjects(response, EntityHierarchy.class); + + isChild = + domains.stream() + .filter(domain -> "rootDomain".equals(domain.getName())) + .findFirst() + .map( + root -> + root.getChildren().stream() + .filter(domain -> "secondLevelDomain".equals(domain.getName())) + .flatMap(secondLevel -> secondLevel.getChildren().stream()) + .anyMatch(thirdLevel -> "thirdLevelDomain".equals(thirdLevel.getName()))) + .orElse(false); + + assertTrue( + isChild, + "thirdLevelDomain should be a child of secondLevelDomain, which should be a child of rootDomain"); + } + + @Test + void get_hierarchicalListOfDomain(TestInfo test) throws HttpResponseException { + Domain rootDomain = createEntity(createRequest("A_ROOT_DOMAIN"), ADMIN_AUTH_HEADERS); + Domain subDomain1 = + createEntity( + createRequest("A_subDomain1").withParent(rootDomain.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS); + Domain subDomain2 = + createEntity( + createRequest("A_subDomain2").withParent(rootDomain.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS); + Domain subDomain3 = + createEntity( + createRequest("A_subDomain3").withParent(rootDomain.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS); + + // Ensure parent has all the newly created children + rootDomain = getEntity(rootDomain.getId(), "children,parent", ADMIN_AUTH_HEADERS); + assertEntityReferences( + new ArrayList<>( + List.of( + subDomain1.getEntityReference(), + subDomain2.getEntityReference(), + subDomain3.getEntityReference())), + rootDomain.getChildren()); + + Domain subSubDomain1 = + createEntity( + createRequest("A_subSubDomain11").withParent(subDomain1.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS); + Domain subSubDomain2 = + createEntity( + createRequest("A_subSubDomain12").withParent(subDomain1.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS); + Domain subSubDomain3 = + createEntity( + createRequest("A_subSubDomain13").withParent(subDomain1.getFullyQualifiedName()), + ADMIN_AUTH_HEADERS); + + // Ensure parent has all the newly created children + subDomain1 = getEntity(subDomain1.getId(), "children,parent", ADMIN_AUTH_HEADERS); + assertEntityReferences( + new ArrayList<>( + List.of( + subSubDomain1.getEntityReference(), + subSubDomain2.getEntityReference(), + subSubDomain3.getEntityReference())), + subDomain1.getChildren()); + assertParent(subSubDomain1, subDomain1.getEntityReference()); + assertParent(subSubDomain2, subDomain1.getEntityReference()); + assertParent(subSubDomain3, subDomain1.getEntityReference()); + + // Create another root domain without hierarchy + Domain secondRootDomain = createEntity(createRequest("B_ROOT_DOMAIN"), ADMIN_AUTH_HEADERS); + + List hierarchyList = getDomainsHierarchy(ADMIN_AUTH_HEADERS).getData(); + + UUID rootDomainId = rootDomain.getId(); + UUID subDomain1Id = subDomain1.getId(); + UUID subDomain2Id = subDomain2.getId(); + UUID subDomain3Id = subDomain3.getId(); + UUID subSubDomain1Id = subSubDomain1.getId(); + UUID subSubDomain2Id = subSubDomain2.getId(); + UUID subSubDomain3Id = subSubDomain3.getId(); + UUID secondRootDomainId = secondRootDomain.getId(); + + EntityHierarchy rootHierarchy = + hierarchyList.stream().filter(h -> h.getId().equals(rootDomainId)).findAny().orElse(null); + assertNotNull(rootHierarchy); + assertEquals(3, rootHierarchy.getChildren().size()); + + List rootChildren = rootHierarchy.getChildren(); + assertEquals(3, rootChildren.size()); + assertTrue(rootChildren.stream().anyMatch(h -> h.getId().equals(subDomain1Id))); + assertTrue(rootChildren.stream().anyMatch(h -> h.getId().equals(subDomain2Id))); + assertTrue(rootChildren.stream().anyMatch(h -> h.getId().equals(subDomain3Id))); + + EntityHierarchy subDomain1Hierarchy = + rootChildren.stream().filter(h -> h.getId().equals(subDomain1Id)).findAny().orElse(null); + assertNotNull(subDomain1Hierarchy); + assertEquals(3, subDomain1Hierarchy.getChildren().size()); + + List subDomain1Children = subDomain1Hierarchy.getChildren(); + assertTrue(subDomain1Children.stream().anyMatch(h -> h.getId().equals(subSubDomain1Id))); + assertTrue(subDomain1Children.stream().anyMatch(h -> h.getId().equals(subSubDomain2Id))); + assertTrue(subDomain1Children.stream().anyMatch(h -> h.getId().equals(subSubDomain3Id))); + + EntityHierarchy subSubDomain1Hierarchy = + subDomain1Children.stream() + .filter(h -> h.getId().equals(subSubDomain1Id)) + .findAny() + .orElse(null); + assertNotNull(subSubDomain1Hierarchy); + assertEquals(0, subSubDomain1Hierarchy.getChildren().size()); + + EntityHierarchy subSubDomain2Hierarchy = + subDomain1Children.stream() + .filter(h -> h.getId().equals(subSubDomain2Id)) + .findAny() + .orElse(null); + assertNotNull(subSubDomain2Hierarchy); + assertEquals(0, subSubDomain2Hierarchy.getChildren().size()); + + EntityHierarchy subSubDomain3Hierarchy = + subDomain1Children.stream() + .filter(h -> h.getId().equals(subSubDomain3Id)) + .findAny() + .orElse(null); + assertNotNull(subSubDomain3Hierarchy); + assertEquals(0, subSubDomain3Hierarchy.getChildren().size()); + + // Verify the new root domain without hierarchy + EntityHierarchy secondRootDomainHierarchy = + hierarchyList.stream() + .filter(h -> h.getId().equals(secondRootDomainId)) + .findAny() + .orElse(null); + assertNotNull(secondRootDomainHierarchy); + assertEquals(0, secondRootDomainHierarchy.getChildren().size()); + } + + private void assertParent(Domain domain, EntityReference expectedParent) + throws HttpResponseException { + assertEquals(expectedParent, domain.getParent()); + // Ensure the parent has the given domain as a child + Domain parent = getEntity(expectedParent.getId(), "children", ADMIN_AUTH_HEADERS); + assertEntityReferencesContain(parent.getChildren(), domain.getEntityReference()); + } + + private EntityHierarchyList getDomainsHierarchy(Map authHeaders) + throws HttpResponseException { + WebTarget target = getResource("domains/hierarchy"); + target = target.queryParam("limit", 25); + return TestUtils.get(target, EntityHierarchyList.class, authHeaders); + } + @Override public CreateDomain createRequest(String name) { return new CreateDomain() From bdcbfcb57ed8da434456c6d9c2b5906b190bc513 Mon Sep 17 00:00:00 2001 From: sonikashah Date: Mon, 6 Jan 2025 10:32:06 +0530 Subject: [PATCH 2/2] use linkedHashmap to maintain order of results in hierarchy --- .../service/search/elasticsearch/ElasticSearchClient.java | 4 +++- .../service/search/opensearch/OpenSearchClient.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 402ce6311efd..8c1974d14be2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -688,7 +688,9 @@ public List buildDomainSearchHierarchy(SearchResponse searchRes entity -> { entity.setChildren(new ArrayList<>()); return entity; - })); + }, + (existing, replacement) -> existing, + LinkedHashMap::new)); List rootDomains = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 2bc2b1375039..f1cbc2e6e568 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -676,7 +676,9 @@ public List buildDomainSearchHierarchy(SearchResponse searchRes entity -> { entity.setChildren(new ArrayList<>()); return entity; - })); + }, + (existing, replacement) -> existing, + LinkedHashMap::new)); List rootDomains = new ArrayList<>();