Skip to content

Commit

Permalink
fix the search for custom properties in advance search for some types (
Browse files Browse the repository at this point in the history
…#19113)

* fix the search in advance search custom properties

* modify v1/metadata/types/customProperties api to get customPropertyConfig along with name and type

* added config list in the entity ref list

* fix type of duration, timestamp and other types

* minor ui changes

* remove customProperty type which are not supported for now

* added playwright test around the duration type property

* fix flaky test

---------

Co-authored-by: sonikashah <[email protected]>
  • Loading branch information
Ashish8689 and sonika-shah authored Jan 8, 2025
1 parent 17ceb63 commit 2179b43
Show file tree
Hide file tree
Showing 13 changed files with 377 additions and 54 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
@Slf4j
public class SchemaFieldExtractor {

private static final Map<String, Map<String, String>> entityFieldsCache =
private static final Map<String, Map<String, FieldDefinition>> entityFieldsCache =
new ConcurrentHashMap<>();

public SchemaFieldExtractor() {
Expand All @@ -55,7 +55,7 @@ private static void initializeEntityFieldsCache() {
Schema mainSchema = loadMainSchema(schemaPath, entityType, schemaUri, schemaClient);

// Extract fields from the schema
Map<String, String> fieldTypesMap = new LinkedHashMap<>();
Map<String, FieldDefinition> fieldTypesMap = new LinkedHashMap<>();
Deque<Schema> processingStack = new ArrayDeque<>();
Set<String> processedFields = new HashSet<>();
extractFieldsFromSchema(mainSchema, "", fieldTypesMap, processingStack, processedFields);
Expand All @@ -75,7 +75,7 @@ public List<FieldDefinition> extractFields(Type typeEntity, String entityType) {
SchemaClient schemaClient = new CustomSchemaClient(schemaUri);
Deque<Schema> processingStack = new ArrayDeque<>();
Set<String> processedFields = new HashSet<>();
Map<String, String> fieldTypesMap = entityFieldsCache.get(entityType);
Map<String, FieldDefinition> fieldTypesMap = entityFieldsCache.get(entityType);
addCustomProperties(
typeEntity, schemaUri, schemaClient, fieldTypesMap, processingStack, processedFields);
return convertMapToFieldList(fieldTypesMap);
Expand All @@ -90,7 +90,7 @@ public Map<String, List<FieldDefinition>> extractAllCustomProperties(
SchemaClient schemaClient = new CustomSchemaClient(schemaUri);
EntityUtil.Fields fieldsParam = new EntityUtil.Fields(Set.of("customProperties"));
Type typeEntity = repository.getByName(uriInfo, entityType, fieldsParam, Include.ALL, false);
Map<String, String> fieldTypesMap = new LinkedHashMap<>();
Map<String, FieldDefinition> fieldTypesMap = new LinkedHashMap<>();
Set<String> processedFields = new HashSet<>();
Deque<Schema> processingStack = new ArrayDeque<>();
addCustomProperties(
Expand Down Expand Up @@ -170,7 +170,7 @@ private static Schema loadMainSchema(
private static void extractFieldsFromSchema(
Schema schema,
String parentPath,
Map<String, String> fieldTypesMap,
Map<String, FieldDefinition> fieldTypesMap,
Deque<Schema> processingStack,
Set<String> processedFields) {
if (processingStack.contains(schema)) {
Expand Down Expand Up @@ -206,7 +206,8 @@ private static void extractFieldsFromSchema(
arraySchema, fullFieldName, fieldTypesMap, processingStack, processedFields);
} else {
String fieldType = mapSchemaTypeToSimpleType(fieldSchema);
fieldTypesMap.putIfAbsent(fullFieldName, fieldType);
fieldTypesMap.putIfAbsent(
fullFieldName, new FieldDefinition(fullFieldName, fieldType, null));
processedFields.add(fullFieldName);
LOG.debug("Added field '{}', Type: '{}'", fullFieldName, fieldType);
// Recursively process nested objects or arrays
Expand All @@ -220,7 +221,7 @@ private static void extractFieldsFromSchema(
handleArraySchema(arraySchema, parentPath, fieldTypesMap, processingStack, processedFields);
} else {
String fieldType = mapSchemaTypeToSimpleType(schema);
fieldTypesMap.putIfAbsent(parentPath, fieldType);
fieldTypesMap.putIfAbsent(parentPath, new FieldDefinition(parentPath, fieldType, null));
LOG.debug("Added field '{}', Type: '{}'", parentPath, fieldType);
}
} finally {
Expand All @@ -231,15 +232,16 @@ private static void extractFieldsFromSchema(
private static void handleReferenceSchema(
ReferenceSchema referenceSchema,
String fullFieldName,
Map<String, String> fieldTypesMap,
Map<String, FieldDefinition> fieldTypesMap,
Deque<Schema> processingStack,
Set<String> processedFields) {

String refUri = referenceSchema.getReferenceValue();
String referenceType = determineReferenceType(refUri);

if (referenceType != null) {
fieldTypesMap.putIfAbsent(fullFieldName, referenceType);
fieldTypesMap.putIfAbsent(
fullFieldName, new FieldDefinition(fullFieldName, referenceType, null));
processedFields.add(fullFieldName);
LOG.debug("Added field '{}', Type: '{}'", fullFieldName, referenceType);
if (referenceType.startsWith("array<") && referenceType.endsWith(">")) {
Expand All @@ -255,7 +257,7 @@ private static void handleReferenceSchema(
referredSchema, fullFieldName, fieldTypesMap, processingStack, processedFields);
}
} else {
fieldTypesMap.putIfAbsent(fullFieldName, "object");
fieldTypesMap.putIfAbsent(fullFieldName, new FieldDefinition(fullFieldName, "object", null));
processedFields.add(fullFieldName);
LOG.debug("Added field '{}', Type: 'object'", fullFieldName);
extractFieldsFromSchema(
Expand All @@ -270,7 +272,7 @@ private static void handleReferenceSchema(
private static void handleArraySchema(
ArraySchema arraySchema,
String fullFieldName,
Map<String, String> fieldTypesMap,
Map<String, FieldDefinition> fieldTypesMap,
Deque<Schema> processingStack,
Set<String> processedFields) {

Expand All @@ -282,7 +284,8 @@ private static void handleArraySchema(

if (itemsReferenceType != null) {
String arrayFieldType = "array<" + itemsReferenceType + ">";
fieldTypesMap.putIfAbsent(fullFieldName, arrayFieldType);
fieldTypesMap.putIfAbsent(
fullFieldName, new FieldDefinition(fullFieldName, arrayFieldType, null));
processedFields.add(fullFieldName);
LOG.debug("Added field '{}', Type: '{}'", fullFieldName, arrayFieldType);
Schema referredItemsSchema = itemsReferenceSchema.getReferredSchema();
Expand All @@ -292,7 +295,8 @@ private static void handleArraySchema(
}
}
String arrayType = mapSchemaTypeToSimpleType(itemsSchema);
fieldTypesMap.putIfAbsent(fullFieldName, "array<" + arrayType + ">");
fieldTypesMap.putIfAbsent(
fullFieldName, new FieldDefinition(fullFieldName, "array<" + arrayType + ">", null));
processedFields.add(fullFieldName);
LOG.debug("Added field '{}', Type: 'array<{}>'", fullFieldName, arrayType);

Expand All @@ -306,7 +310,7 @@ private void addCustomProperties(
Type typeEntity,
String schemaUri,
SchemaClient schemaClient,
Map<String, String> fieldTypesMap,
Map<String, FieldDefinition> fieldTypesMap,
Deque<Schema> processingStack,
Set<String> processedFields) {
if (typeEntity == null || typeEntity.getCustomProperties() == null) {
Expand All @@ -320,9 +324,13 @@ private void addCustomProperties(

LOG.debug("Processing custom property '{}'", fullFieldName);

Object customPropertyConfigObj = customProperty.getCustomPropertyConfig();

if (isEntityReferenceList(propertyType)) {
String referenceType = "array<entityReference>";
fieldTypesMap.putIfAbsent(fullFieldName, referenceType);
FieldDefinition referenceFieldDefinition =
new FieldDefinition(fullFieldName, referenceType, customPropertyConfigObj);
fieldTypesMap.putIfAbsent(fullFieldName, referenceFieldDefinition);
processedFields.add(fullFieldName);
LOG.debug("Added custom property '{}', Type: '{}'", fullFieldName, referenceType);

Expand All @@ -337,7 +345,9 @@ private void addCustomProperties(
}
} else if (isEntityReference(propertyType)) {
String referenceType = "entityReference";
fieldTypesMap.putIfAbsent(fullFieldName, referenceType);
FieldDefinition referenceFieldDefinition =
new FieldDefinition(fullFieldName, referenceType, customPropertyConfigObj);
fieldTypesMap.putIfAbsent(fullFieldName, referenceFieldDefinition);
processedFields.add(fullFieldName);
LOG.debug("Added custom property '{}', Type: '{}'", fullFieldName, referenceType);

Expand All @@ -351,17 +361,22 @@ private void addCustomProperties(
fullFieldName);
}
} else {
fieldTypesMap.putIfAbsent(fullFieldName, propertyType);
FieldDefinition entityFieldDefinition =
new FieldDefinition(fullFieldName, propertyType, customPropertyConfigObj);
fieldTypesMap.putIfAbsent(fullFieldName, entityFieldDefinition);
processedFields.add(fullFieldName);
LOG.debug("Added custom property '{}', Type: '{}'", fullFieldName, propertyType);
}
}
}

private List<FieldDefinition> convertMapToFieldList(Map<String, String> fieldTypesMap) {
private List<FieldDefinition> convertMapToFieldList(Map<String, FieldDefinition> fieldTypesMap) {
List<FieldDefinition> fieldsList = new ArrayList<>();
for (Map.Entry<String, String> entry : fieldTypesMap.entrySet()) {
fieldsList.add(new FieldDefinition(entry.getKey(), entry.getValue()));
for (Map.Entry<String, FieldDefinition> entry : fieldTypesMap.entrySet()) {
FieldDefinition fieldDef = entry.getValue();
fieldsList.add(
new FieldDefinition(
fieldDef.getName(), fieldDef.getType(), fieldDef.getCustomPropertyConfig()));
}
return fieldsList;
}
Expand Down Expand Up @@ -622,10 +637,12 @@ private String mapUrlToResourcePath(String url) {
public static class FieldDefinition {
private String name;
private String type;
private Object customPropertyConfig;

public FieldDefinition(String name, String type) {
public FieldDefinition(String name, String type, Object customPropertyConfig) {
this.name = name;
this.type = type;
this.customPropertyConfig = customPropertyConfig;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import test, { expect } from '@playwright/test';
import { CUSTOM_PROPERTIES_ENTITIES } from '../../constant/customProperty';
import { GlobalSettingOptions } from '../../constant/settings';
import { SidebarItem } from '../../constant/sidebar';
import { TableClass } from '../../support/entity/TableClass';
import {
selectOption,
showAdvancedSearchDialog,
} from '../../utils/advancedSearch';
import { advanceSearchSaveFilter } from '../../utils/advancedSearchCustomProperty';
import { createNewPage, redirectToHomePage, uuid } from '../../utils/common';
import { addCustomPropertiesForEntity } from '../../utils/customProperty';
import { settingClick, sidebarClick } from '../../utils/sidebar';

// use the admin user to login
test.use({ storageState: 'playwright/.auth/admin.json' });

test.describe('Advanced Search Custom Property', () => {
const table = new TableClass();
const durationPropertyName = `pwCustomPropertyDurationTest${uuid()}`;
const durationPropertyValue = 'PT1H30M';

test.beforeAll('Setup pre-requests', async ({ browser }) => {
const { apiContext, afterAction } = await createNewPage(browser);
await table.create(apiContext);
await afterAction();
});

test.afterAll('Cleanup', async ({ browser }) => {
const { apiContext, afterAction } = await createNewPage(browser);
await table.delete(apiContext);
await afterAction();
});

test('Create, Assign and Test Advance Search for Duration', async ({
page,
}) => {
test.slow(true);

await redirectToHomePage(page);

await test.step('Create and Assign Custom Property Value', async () => {
await settingClick(page, GlobalSettingOptions.TABLES, true);

await addCustomPropertiesForEntity({
page,
propertyName: durationPropertyName,
customPropertyData: CUSTOM_PROPERTIES_ENTITIES['entity_table'],
customType: 'Duration',
});

await table.visitEntityPage(page);

await page.getByTestId('custom_properties').click(); // Tab Click

await page
.getByTestId(`custom-property-${durationPropertyName}-card`)
.locator('svg')
.click(); // Add Custom Property Value

await page.getByTestId('duration-input').fill(durationPropertyValue);

const saveResponse = page.waitForResponse('/api/v1/tables/*');
await page.getByTestId('inline-save-btn').click();
await saveResponse;
});

await test.step('Verify Duration Type in Advance Search ', async () => {
await sidebarClick(page, SidebarItem.EXPLORE);

await showAdvancedSearchDialog(page);

const ruleLocator = page.locator('.rule').nth(0);

// Perform click on rule field
await selectOption(
page,
ruleLocator.locator('.rule--field .ant-select'),
'Custom Properties'
);

// Perform click on custom property type to filter
await selectOption(
page,
ruleLocator.locator('.rule--field .ant-select'),
durationPropertyName
);

const inputElement = ruleLocator.locator(
'.rule--widget--TEXT input[type="text"]'
);

await inputElement.fill(durationPropertyValue);

await advanceSearchSaveFilter(page, durationPropertyValue);

await expect(
page.getByTestId(
`table-data-card_${table.entityResponseData.fullyQualifiedName}`
)
).toBeVisible();

// Check around the Partial Search Value
const partialSearchValue = durationPropertyValue.slice(0, 3);

await page.getByTestId('advance-search-filter-btn').click();

await expect(page.locator('[role="dialog"].ant-modal')).toBeVisible();

// Perform click on operator
await selectOption(
page,
ruleLocator.locator('.rule--operator .ant-select'),
'Contains'
);

await inputElement.fill(partialSearchValue);

await advanceSearchSaveFilter(page, partialSearchValue);

await expect(
page.getByTestId(
`table-data-card_${table.entityResponseData.fullyQualifiedName}`
)
).toBeVisible();
});
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { expect, Page } from '@playwright/test';

export const advanceSearchSaveFilter = async (
page: Page,
propertyValue: string
) => {
const searchResponse = page.waitForResponse(
'/api/v1/search/query?*index=dataAsset&from=0&size=10*'
);
await page.getByTestId('apply-btn').click();

const res = await searchResponse;
const json = await res.json();

expect(JSON.stringify(json)).toContain(propertyValue);
};
Loading

0 comments on commit 2179b43

Please sign in to comment.