From dc4b2c9b472a87ba1e093c070954c7c081dd5c77 Mon Sep 17 00:00:00 2001 From: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com> Date: Mon, 2 Dec 2024 12:26:30 +0530 Subject: [PATCH 1/8] Google returns null refresh token on Refresh requests (#18868) --- .../service/security/AuthenticationCodeFlowHandler.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthenticationCodeFlowHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthenticationCodeFlowHandler.java index a273f746b36d..de95bcce7765 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthenticationCodeFlowHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthenticationCodeFlowHandler.java @@ -916,7 +916,9 @@ private void populateCredentialsFromTokenResponse( OIDCTokenResponse tokenSuccessResponse, OidcCredentials credentials) { OIDCTokens oidcTokens = tokenSuccessResponse.getOIDCTokens(); credentials.setAccessToken(oidcTokens.getAccessToken()); - credentials.setRefreshToken(oidcTokens.getRefreshToken()); + if (oidcTokens.getRefreshToken() != null) { + credentials.setRefreshToken(oidcTokens.getRefreshToken()); + } if (oidcTokens.getIDToken() != null) { credentials.setIdToken(oidcTokens.getIDToken()); } From d5e54e479110c73dec4e2c286d5bbf9e3bdca23d Mon Sep 17 00:00:00 2001 From: Sachin Chaurasiya Date: Mon, 2 Dec 2024 13:04:50 +0530 Subject: [PATCH 2/8] Minor: improve the CSV import encoding for column description and glossaryTerms (#18870) --- .../src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx index f79facd626ff..3791e8f4019e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSVUtilsClassBase.tsx @@ -44,6 +44,8 @@ class CSVUtilsClassBase { 'description', 'glossaryTerms', 'relatedTerms', + 'column.description', + 'column.glossaryTerms', ]; } From bb1533134082987f5bc0cee1e82338a516d292c4 Mon Sep 17 00:00:00 2001 From: RounakDhillon <162090200+RounakDhillon@users.noreply.github.com> Date: Mon, 2 Dec 2024 13:09:59 +0530 Subject: [PATCH 3/8] Docs: Updating Collate in Pipeline Status Widget (#18869) Co-authored-by: Rounak Dhillon --- openmetadata-docs/content/v1.5.x/releases/releases/index.md | 2 +- .../content/v1.6.x-SNAPSHOT/releases/releases/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openmetadata-docs/content/v1.5.x/releases/releases/index.md b/openmetadata-docs/content/v1.5.x/releases/releases/index.md index 9bcc24720733..34ea08817785 100644 --- a/openmetadata-docs/content/v1.5.x/releases/releases/index.md +++ b/openmetadata-docs/content/v1.5.x/releases/releases/index.md @@ -449,7 +449,7 @@ We are also making the discovery of data more accessible for users introducing a {% youtube videoId="45ekUIRO1Ec" start="0:00" end="1:11" width="560px" height="315px" /%} -## Pipeline Status Widget +## Pipeline Status Widget (Collate) We are also adding another widget you can use to customize the Landing Page of the User Personas in your organization. diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/releases/releases/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/releases/releases/index.md index 9bcc24720733..34ea08817785 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/releases/releases/index.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/releases/releases/index.md @@ -449,7 +449,7 @@ We are also making the discovery of data more accessible for users introducing a {% youtube videoId="45ekUIRO1Ec" start="0:00" end="1:11" width="560px" height="315px" /%} -## Pipeline Status Widget +## Pipeline Status Widget (Collate) We are also adding another widget you can use to customize the Landing Page of the User Personas in your organization. From 17dd0ee4b956690e18466ecc3abc7502f0b47eaf Mon Sep 17 00:00:00 2001 From: Sweta Agarwalla <105535990+sweta1308@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:42:19 +0530 Subject: [PATCH 4/8] update to reorder domain and owner tags (#18872) --- .../ExploreSearchCard/ExploreSearchCard.tsx | 82 ++++++++++--------- 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.tsx index 99259365e8a0..2a61c7a38156 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.tsx @@ -78,54 +78,56 @@ const ExploreSearchCard: React.FC = forwardRef< ); const _otherDetails: ExtraInfo[] = [ + ...(source?.domain + ? [ + { + key: 'Domain', + value: getDomainPath(source.domain.fullyQualifiedName), + placeholderText: getEntityName(source.domain), + isLink: true, + openInNewTab: false, + }, + ] + : !searchClassBase + .getListOfEntitiesWithoutDomain() + .includes(source?.entityType ?? '') + ? [ + { + key: 'Domain', + value: '', + }, + ] + : []), + { key: 'Owner', value: ( - + ), }, - ]; - - if (source?.domain) { - const domain = getEntityName(source.domain); - const domainLink = getDomainPath(source.domain.fullyQualifiedName); - _otherDetails.push({ - key: 'Domain', - value: domainLink, - placeholderText: domain, - isLink: true, - openInNewTab: false, - }); - } else { - const entitiesWithoutDomain = - searchClassBase.getListOfEntitiesWithoutDomain(); - if (!entitiesWithoutDomain.includes(source.entityType ?? '')) { - _otherDetails.push({ - key: 'Domain', - value: '', - }); - } - } - if ( - !searchClassBase + ...(!searchClassBase .getListOfEntitiesWithoutTier() - .includes((source.entityType ?? '') as EntityType) - ) { - _otherDetails.push({ - key: 'Tier', - value: tierValue, - }); - } + .includes((source?.entityType ?? '') as EntityType) + ? [ + { + key: 'Tier', + value: tierValue, + }, + ] + : []), - if ('usageSummary' in source) { - _otherDetails.push({ - value: getUsagePercentile( - source.usageSummary?.weeklyStats?.percentileRank ?? 0, - true - ), - }); - } + ...('usageSummary' in source + ? [ + { + value: getUsagePercentile( + source.usageSummary?.weeklyStats?.percentileRank ?? 0, + true + ), + }, + ] + : []), + ]; return _otherDetails; }, [source]); From a72b293bda94dc1072732c51fc6b8aa4e58c26a8 Mon Sep 17 00:00:00 2001 From: Farreeda <94711213+Farreeda@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:44:53 +0200 Subject: [PATCH 5/8] Fixes issue #18781: Adding lambda permissions to Athena connector documentation (#18871) * Adding lambda permissions to Athena connector documentation * Add lambda permissions to athena connector documentation yaml file * Adding lambda permissions to Athena doc v1.5 * Adding lambda permissions for Athena connector document v1.4.x --- .../connectors/database/athena/index.md | 19 +++++++++++++++++++ .../v1.4.x/connectors/database/athena/yaml.md | 19 +++++++++++++++++++ .../connectors/database/athena/index.md | 19 +++++++++++++++++++ .../v1.5.x/connectors/database/athena/yaml.md | 19 +++++++++++++++++++ .../connectors/database/athena/index.md | 19 +++++++++++++++++++ .../connectors/database/athena/yaml.md | 19 +++++++++++++++++++ 6 files changed, 114 insertions(+) diff --git a/openmetadata-docs/content/v1.4.x/connectors/database/athena/index.md b/openmetadata-docs/content/v1.4.x/connectors/database/athena/index.md index 52b56b9024c8..65329e093431 100644 --- a/openmetadata-docs/content/v1.4.x/connectors/database/athena/index.md +++ b/openmetadata-docs/content/v1.4.x/connectors/database/athena/index.md @@ -105,6 +105,25 @@ And is defined as: "arn:aws:athena:<>:<>:datacatalog/<>/database/<>/table/<>" "arn:aws:athena:<>:<>:datacatalog/<>/database/<>/table/<>/column/<>" ] + }, + { + "Action": [ + "lambda:InvokeFunction" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:lambda:<>:<>:function:<>" + ] + }, + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:DescribeKey" + ], + "Resource": [ + "arn:aws:kms:<>:<>:key/<>" + ] } ] } diff --git a/openmetadata-docs/content/v1.4.x/connectors/database/athena/yaml.md b/openmetadata-docs/content/v1.4.x/connectors/database/athena/yaml.md index a43eb844373e..74ba2e9c0578 100644 --- a/openmetadata-docs/content/v1.4.x/connectors/database/athena/yaml.md +++ b/openmetadata-docs/content/v1.4.x/connectors/database/athena/yaml.md @@ -103,6 +103,25 @@ And is defined as: "arn:aws:athena:<>:<>:datacatalog/<>/database/<>/table/<>" "arn:aws:athena:<>:<>:datacatalog/<>/database/<>/table/<>/column/<>" ] + }, + { + "Action": [ + "lambda:InvokeFunction" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:lambda:<>:<>:function:<>" + ] + }, + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:DescribeKey" + ], + "Resource": [ + "arn:aws:kms:<>:<>:key/<>" + ] } ] } diff --git a/openmetadata-docs/content/v1.5.x/connectors/database/athena/index.md b/openmetadata-docs/content/v1.5.x/connectors/database/athena/index.md index a02b6b899598..fac0269c0f4a 100644 --- a/openmetadata-docs/content/v1.5.x/connectors/database/athena/index.md +++ b/openmetadata-docs/content/v1.5.x/connectors/database/athena/index.md @@ -105,6 +105,25 @@ And is defined as: "arn:aws:athena:<>:<>:datacatalog/<>/database/<>/table/<>" "arn:aws:athena:<>:<>:datacatalog/<>/database/<>/table/<>/column/<>" ] + }, + { + "Action": [ + "lambda:InvokeFunction" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:lambda:<>:<>:function:<>" + ] + }, + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:DescribeKey" + ], + "Resource": [ + "arn:aws:kms:<>:<>:key/<>" + ] } ] } diff --git a/openmetadata-docs/content/v1.5.x/connectors/database/athena/yaml.md b/openmetadata-docs/content/v1.5.x/connectors/database/athena/yaml.md index 540480f77c42..bfade366ddeb 100644 --- a/openmetadata-docs/content/v1.5.x/connectors/database/athena/yaml.md +++ b/openmetadata-docs/content/v1.5.x/connectors/database/athena/yaml.md @@ -103,6 +103,25 @@ And is defined as: "arn:aws:athena:<>:<>:datacatalog/<>/database/<>/table/<>" "arn:aws:athena:<>:<>:datacatalog/<>/database/<>/table/<>/column/<>" ] + }, + { + "Action": [ + "lambda:InvokeFunction" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:lambda:<>:<>:function:<>" + ] + }, + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:DescribeKey" + ], + "Resource": [ + "arn:aws:kms:<>:<>:key/<>" + ] } ] } diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/athena/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/athena/index.md index a02b6b899598..fac0269c0f4a 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/athena/index.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/athena/index.md @@ -105,6 +105,25 @@ And is defined as: "arn:aws:athena:<>:<>:datacatalog/<>/database/<>/table/<>" "arn:aws:athena:<>:<>:datacatalog/<>/database/<>/table/<>/column/<>" ] + }, + { + "Action": [ + "lambda:InvokeFunction" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:lambda:<>:<>:function:<>" + ] + }, + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:DescribeKey" + ], + "Resource": [ + "arn:aws:kms:<>:<>:key/<>" + ] } ] } diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/athena/yaml.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/athena/yaml.md index 540480f77c42..bfade366ddeb 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/athena/yaml.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/athena/yaml.md @@ -103,6 +103,25 @@ And is defined as: "arn:aws:athena:<>:<>:datacatalog/<>/database/<>/table/<>" "arn:aws:athena:<>:<>:datacatalog/<>/database/<>/table/<>/column/<>" ] + }, + { + "Action": [ + "lambda:InvokeFunction" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:lambda:<>:<>:function:<>" + ] + }, + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:DescribeKey" + ], + "Resource": [ + "arn:aws:kms:<>:<>:key/<>" + ] } ] } From f3f2efe3d58c8206873efe143164b18e061f0fb8 Mon Sep 17 00:00:00 2001 From: Shailesh Parmar Date: Mon, 2 Dec 2024 19:24:25 +0530 Subject: [PATCH 6/8] Minor: fixed connection modal style (#18860) * Minor: fixed connection modal style * addressing comment --- .../IngestionRunDetailsModal.tsx | 46 ++++++++++--------- .../connection-step-card.less | 1 - 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/IngestionRunDetailsModal/IngestionRunDetailsModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/IngestionRunDetailsModal/IngestionRunDetailsModal.tsx index bfd31753d2df..e7ff9742351c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/IngestionRunDetailsModal/IngestionRunDetailsModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/IngestionRunDetailsModal/IngestionRunDetailsModal.tsx @@ -11,7 +11,7 @@ * limitations under the License. */ -import { Button, Modal, Typography } from 'antd'; +import { Button, Col, Modal, Row, Typography } from 'antd'; import { ColumnType } from 'antd/lib/table'; import { ExpandableConfig } from 'antd/lib/table/interface'; import { startCase } from 'lodash'; @@ -87,26 +87,30 @@ function IngestionRunDetailsModal({ const expandable: ExpandableConfig = useMemo( () => ({ expandedRowRender: (record) => { - return ( - record.failures?.map((failure) => ( - - )) ?? [] - ); + return record.failures ? ( + + {record.failures.map((failure) => ( + + + + ))} + + ) : undefined; }, indentSize: 0, expandIcon: () => null, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/TestConnection/ConnectionStepCard/connection-step-card.less b/openmetadata-ui/src/main/resources/ui/src/components/common/TestConnection/ConnectionStepCard/connection-step-card.less index f9acecf32155..6de52463b5f1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/TestConnection/ConnectionStepCard/connection-step-card.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/TestConnection/ConnectionStepCard/connection-step-card.less @@ -24,7 +24,6 @@ box-sizing: border-box; border: 1px solid @border-color; border-radius: 4px; - margin-top: 16px; &.success { border-color: #28a7454d; } From 244a933adcea70714daae4d869308ff96c80f043 Mon Sep 17 00:00:00 2001 From: sonika-shah <58761340+sonika-shah@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:12:47 +0530 Subject: [PATCH 7/8] Minor : Fix #18775 validation on enum custom property (#18855) * Minor : Fix #18775 validation on enum custom property * fix tests * Add tests --- .../java/org/openmetadata/csv/EntityCsv.java | 34 ++++++++- .../service/jdbi3/EntityRepository.java | 22 ++++++ .../glossary/GlossaryResourceTest.java | 75 ++++++++++++++++++- 3 files changed, 123 insertions(+), 8 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 0b2e0ccb5e93..b8e1e879deb2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -391,10 +391,15 @@ private void validateExtension( fieldValue.toString(), customPropertyType, propertyConfig); - case "enum" -> { - List enumKeys = listOrEmpty(fieldToInternalArray(fieldValue.toString())); - fieldValue = enumKeys.isEmpty() ? null : enumKeys; - } + case "enum" -> fieldValue = + parseEnumType( + printer, + csvRecord, + fieldNumber, + fieldName, + customPropertyType, + fieldValue, + propertyConfig); case "timeInterval" -> fieldValue = parseTimeInterval(printer, csvRecord, fieldNumber, fieldName, fieldValue); case "number", "integer", "timestamp" -> fieldValue = @@ -573,6 +578,27 @@ private Object parseTableType( return tableJson; } + private Object parseEnumType( + CSVPrinter printer, + CSVRecord csvRecord, + int fieldNumber, + String fieldName, + String customPropertyType, + Object fieldValue, + String propertyConfig) + throws IOException { + List enumKeys = listOrEmpty(fieldToInternalArray(fieldValue.toString())); + try { + EntityRepository.validateEnumKeys(fieldName, JsonUtils.valueToTree(enumKeys), propertyConfig); + } catch (Exception e) { + importFailure( + printer, + invalidCustomPropertyValue(fieldNumber, fieldName, customPropertyType, e.getMessage()), + csvRecord); + } + return enumKeys.isEmpty() ? null : enumKeys; + } + private void validateAndUpdateExtension( CSVPrinter printer, CSVRecord csvRecord, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 9a7cda3fa51a..f4a87d6d7389 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -154,6 +154,7 @@ import org.openmetadata.schema.type.api.BulkOperationResult; import org.openmetadata.schema.type.api.BulkResponse; import org.openmetadata.schema.type.csv.CsvImportResult; +import org.openmetadata.schema.type.customProperties.EnumConfig; import org.openmetadata.schema.type.customProperties.TableConfig; import org.openmetadata.schema.utils.EntityInterfaceUtil; import org.openmetadata.service.Entity; @@ -1526,6 +1527,7 @@ private void validateAndUpdateExtensionBasedOnPropertyType( jsonNode.put(fieldName, formattedValue); } case "table-cp" -> validateTableType(fieldValue, propertyConfig, fieldName); + case "enum" -> validateEnumKeys(fieldName, fieldValue, propertyConfig); default -> {} } } @@ -1608,6 +1610,26 @@ private void validateTableType(JsonNode fieldValue, String propertyConfig, Strin } } + public static void validateEnumKeys( + String fieldName, JsonNode fieldValue, String propertyConfig) { + JsonNode propertyConfigNode = JsonUtils.readTree(propertyConfig); + EnumConfig config = JsonUtils.treeToValue(propertyConfigNode, EnumConfig.class); + + if (!config.getMultiSelect() && fieldValue.size() > 1) { + throw new IllegalArgumentException( + String.format("Only one value allowed for non-multiSelect %s property", fieldName)); + } + Set validValues = new HashSet<>(config.getValues()); + Set fieldValues = new HashSet<>(); + fieldValue.forEach(value -> fieldValues.add(value.asText())); + + if (!validValues.containsAll(fieldValues)) { + fieldValues.removeAll(validValues); + throw new IllegalArgumentException( + String.format("Values '%s' not supported for property %s", fieldValues, fieldName)); + } + } + public final void storeExtension(EntityInterface entity) { JsonNode jsonNode = JsonUtils.valueToTree(entity.getExtension()); Iterator> customFields = jsonNode.fields(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index b04a89c76ecc..37501156f120 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -652,7 +652,7 @@ invalidCustomPropertyKeyRecord, invalidCustomPropertyKey(11, "invalidCustomPrope // Create glossaryTerm with Invalid custom property of type table TableConfig tableConfig = new TableConfig().withColumns(Set.of("columnName1", "columnName2", "columnName3")); - CustomProperty glossaryTermEnumCp = + CustomProperty glossaryTermTableCp = new CustomProperty() .withName("glossaryTermTableCp") .withDescription("table type custom property ") @@ -665,7 +665,7 @@ invalidCustomPropertyKeyRecord, invalidCustomPropertyKey(11, "invalidCustomPrope Entity.GLOSSARY_TERM, "customProperties", ADMIN_AUTH_HEADERS); entityType = typeResourceTest.addAndCheckCustomProperty( - entityType.getId(), glossaryTermEnumCp, OK, ADMIN_AUTH_HEADERS); + entityType.getId(), glossaryTermTableCp, OK, ADMIN_AUTH_HEADERS); String invalidTableTypeRecord = ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,\"glossaryTermTableCp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value,row_1_col4_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value,row_2_col4_Value\""; String invalidTableTypeValue = @@ -686,6 +686,73 @@ invalidCustomPropertyKeyRecord, invalidCustomPropertyKey(11, "invalidCustomPrope "Column count should be less than or equal to " + tableConfig.getMaxColumns())) }; assertRows(result, expectedRows); + + // Create glossaryTerm with invalid multiple values for non multi-select property + CustomProperty glossaryTermSingleSelectEnumCp = + new CustomProperty() + .withName("glossaryTermSingleSelectEnumCp") + .withDescription("non-multi select enum type custom property ") + .withPropertyType(ENUM_TYPE.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + Map.of( + "values", + List.of("singleSelect-1", "singleSelect-2", "singleSelect-3"), + "multiSelect", + false))); + + entityType = + typeResourceTest.getEntityByName( + Entity.GLOSSARY_TERM, "customProperties", ADMIN_AUTH_HEADERS); + entityType = + typeResourceTest.addAndCheckCustomProperty( + entityType.getId(), glossaryTermSingleSelectEnumCp, OK, ADMIN_AUTH_HEADERS); + String invalidEnumTypeRecord = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermSingleSelectEnumCp:any random string"; + String invalidEnumTypeValue = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermSingleSelectEnumCp:any random string"; + csv = createCsv(GlossaryCsv.HEADERS, listOf(invalidEnumTypeValue), null); + result = importCsv(glossaryName, csv, false); + Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + invalidEnumTypeRecord, + invalidCustomPropertyValue( + 11, + "glossaryTermSingleSelectEnumCp", + ENUM_TYPE.getDisplayName(), + String.format( + "Values '[any random string]' not supported for property " + + glossaryTermSingleSelectEnumCp.getName()))) + }; + assertRows(result, expectedRows); + + String invalidEnumTypeRecord2 = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermSingleSelectEnumCp:singleSelect-1|singleSelect-2"; + String invalidEnumTypeValue2 = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermSingleSelectEnumCp:singleSelect-1|singleSelect-2"; + csv = createCsv(GlossaryCsv.HEADERS, listOf(invalidEnumTypeValue2), null); + result = importCsv(glossaryName, csv, false); + Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + invalidEnumTypeRecord2, + invalidCustomPropertyValue( + 11, + "glossaryTermSingleSelectEnumCp", + ENUM_TYPE.getDisplayName(), + String.format( + "Only one value allowed for non-multiSelect %s property", + glossaryTermSingleSelectEnumCp.getName()))) + }; + assertRows(result, expectedRows); } @Test @@ -844,7 +911,7 @@ void testGlossaryImportExport() throws IOException { ",g1,dsp1,\"dsc1,1\",h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s,\"glossaryTermDateCp:18-09-2024;glossaryTermDateTimeCp:18-09-2024 01:09:34;glossaryTermDurationCp:PT5H30M10S;glossaryTermEmailCp:admin@open-metadata.org;glossaryTermEntRefCp:team:\"\"%s\"\";glossaryTermEntRefListCp:user:\"\"%s\"\"|user:\"\"%s\"\"\"", reviewerRef.get(0), user1, "Approved", team11, user1, user2), String.format( - ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:singleVal1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", + ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:single1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, "Approved"), String.format( "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,", @@ -857,7 +924,7 @@ void testGlossaryImportExport() throws IOException { ",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,PII.None,user:%s,user:%s,%s,\"glossaryTermDateCp:18-09-2024;glossaryTermDateTimeCp:18-09-2024 01:09:34;glossaryTermDurationCp:PT5H30M10S;glossaryTermEmailCp:admin@open-metadata.org;glossaryTermEntRefCp:team:\"\"%s\"\";glossaryTermEntRefListCp:user:\"\"%s\"\"|user:\"\"%s\"\"\"", reviewerRef.get(0), user1, "Approved", team11, user1, user2), String.format( - ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:singleVal1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", + ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:single1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, user2, "Approved"), String.format( "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,\"\"\"glossaryTermTableCol1Cp:row_1_col1_Value,,\"\";\"\"glossaryTermTableCol3Cp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value\"\"\"", From e715a7c26a32a50521315f628fbe861336958b72 Mon Sep 17 00:00:00 2001 From: "Gyowanny P. Queiroz" <81326247+gyohub@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:53:16 -0300 Subject: [PATCH 8/8] Fix typo on the ingestion word (#18881) --- .../contribute/developing-a-new-connector/define-json-schema.md | 2 +- .../contribute/developing-a-new-connector/define-json-schema.md | 2 +- .../contribute/developing-a-new-connector/define-json-schema.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openmetadata-docs/content/v1.4.x/developers/contribute/developing-a-new-connector/define-json-schema.md b/openmetadata-docs/content/v1.4.x/developers/contribute/developing-a-new-connector/define-json-schema.md index 067849818eb8..8067613ee390 100644 --- a/openmetadata-docs/content/v1.4.x/developers/contribute/developing-a-new-connector/define-json-schema.md +++ b/openmetadata-docs/content/v1.4.x/developers/contribute/developing-a-new-connector/define-json-schema.md @@ -363,7 +363,7 @@ Now that you have your Connection defined in the JSON Schema, we can proceed to {%inlineCallout color="violet-70" - bold="Develop the Ingesion Code" + bold="Develop the Ingestion Code" icon="MdArrowForward" href="/developers/contribute/developing-a-new-connector/develop-ingestion-code"%} Learn what you need to implement for the Connector's logic diff --git a/openmetadata-docs/content/v1.5.x/developers/contribute/developing-a-new-connector/define-json-schema.md b/openmetadata-docs/content/v1.5.x/developers/contribute/developing-a-new-connector/define-json-schema.md index 067849818eb8..8067613ee390 100644 --- a/openmetadata-docs/content/v1.5.x/developers/contribute/developing-a-new-connector/define-json-schema.md +++ b/openmetadata-docs/content/v1.5.x/developers/contribute/developing-a-new-connector/define-json-schema.md @@ -363,7 +363,7 @@ Now that you have your Connection defined in the JSON Schema, we can proceed to {%inlineCallout color="violet-70" - bold="Develop the Ingesion Code" + bold="Develop the Ingestion Code" icon="MdArrowForward" href="/developers/contribute/developing-a-new-connector/develop-ingestion-code"%} Learn what you need to implement for the Connector's logic diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/developers/contribute/developing-a-new-connector/define-json-schema.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/developers/contribute/developing-a-new-connector/define-json-schema.md index 067849818eb8..8067613ee390 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/developers/contribute/developing-a-new-connector/define-json-schema.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/developers/contribute/developing-a-new-connector/define-json-schema.md @@ -363,7 +363,7 @@ Now that you have your Connection defined in the JSON Schema, we can proceed to {%inlineCallout color="violet-70" - bold="Develop the Ingesion Code" + bold="Develop the Ingestion Code" icon="MdArrowForward" href="/developers/contribute/developing-a-new-connector/develop-ingestion-code"%} Learn what you need to implement for the Connector's logic