Skip to content

Commit

Permalink
Merge branch 'open-metadata:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
@akashverma0786
akashverma0786 authored Dec 2, 2024
2 parents 8a62446 + e715a7c commit 529cb5f
Show file tree
Hide file tree
Showing 19 changed files with 314 additions and 76 deletions.
19 changes: 19 additions & 0 deletions openmetadata-docs/content/v1.4.x/connectors/database/athena/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,25 @@ And is defined as:
"arn:aws:athena:<<AWS_REGION>>:<<ACCOUNT_ID>>:datacatalog/<<DATA_CATALOG_NAME>>/database/<<DATABASE_NAME>>/table/<<TABLE_NAME>>"
"arn:aws:athena:<<AWS_REGION>>:<<ACCOUNT_ID>>:datacatalog/<<DATA_CATALOG_NAME>>/database/<<DATABASE_NAME>>/table/<<TABLE_NAME>>/column/<<COLUMN_NAME>>"
]
},
{
"Action": [
"lambda:InvokeFunction"
],
"Effect": "Allow",
"Resource": [
"arn:aws:lambda:<<AWS_REGION>>:<<ACCOUNT_ID>>:function:<<CONNECTOR_NAME>>"
]
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:DescribeKey"
],
"Resource": [
"arn:aws:kms:<<AWS_REGION>>:<<ACCOUNT_ID>>:key/<<KMS_KEY_ID>>"
]
}
]
}
Expand Down
19 changes: 19 additions & 0 deletions openmetadata-docs/content/v1.4.x/connectors/database/athena/yaml.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,25 @@ And is defined as:
"arn:aws:athena:<<AWS_REGION>>:<<ACCOUNT_ID>>:datacatalog/<<DATA_CATALOG_NAME>>/database/<<DATABASE_NAME>>/table/<<TABLE_NAME>>"
"arn:aws:athena:<<AWS_REGION>>:<<ACCOUNT_ID>>:datacatalog/<<DATA_CATALOG_NAME>>/database/<<DATABASE_NAME>>/table/<<TABLE_NAME>>/column/<<COLUMN_NAME>>"
]
},
{
"Action": [
"lambda:InvokeFunction"
],
"Effect": "Allow",
"Resource": [
"arn:aws:lambda:<<AWS_REGION>>:<<ACCOUNT_ID>>:function:<<CONNECTOR_NAME>>"
]
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:DescribeKey"
],
"Resource": [
"arn:aws:kms:<<AWS_REGION>>:<<ACCOUNT_ID>>:key/<<KMS_KEY_ID>>"
]
}
]
}
Expand Down
2 changes: 1 addition & 1 deletion ...t/v1.4.x/developers/contribute/developing-a-new-connector/define-json-schema.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ Now that you have your Connection defined in the JSON Schema, we can proceed to

{%inlineCallout
color="violet-70"
bold="Develop the Ingesion Code"
bold="Develop the Ingestion Code"
icon="MdArrowForward"
href="/developers/contribute/developing-a-new-connector/develop-ingestion-code"%}
Learn what you need to implement for the Connector's logic
Expand Down
19 changes: 19 additions & 0 deletions openmetadata-docs/content/v1.5.x/connectors/database/athena/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,25 @@ And is defined as:
"arn:aws:athena:<<AWS_REGION>>:<<ACCOUNT_ID>>:datacatalog/<<DATA_CATALOG_NAME>>/database/<<DATABASE_NAME>>/table/<<TABLE_NAME>>"
"arn:aws:athena:<<AWS_REGION>>:<<ACCOUNT_ID>>:datacatalog/<<DATA_CATALOG_NAME>>/database/<<DATABASE_NAME>>/table/<<TABLE_NAME>>/column/<<COLUMN_NAME>>"
]
},
{
"Action": [
"lambda:InvokeFunction"
],
"Effect": "Allow",
"Resource": [
"arn:aws:lambda:<<AWS_REGION>>:<<ACCOUNT_ID>>:function:<<CONNECTOR_NAME>>"
]
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:DescribeKey"
],
"Resource": [
"arn:aws:kms:<<AWS_REGION>>:<<ACCOUNT_ID>>:key/<<KMS_KEY_ID>>"
]
}
]
}
Expand Down
19 changes: 19 additions & 0 deletions openmetadata-docs/content/v1.5.x/connectors/database/athena/yaml.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,25 @@ And is defined as:
"arn:aws:athena:<<AWS_REGION>>:<<ACCOUNT_ID>>:datacatalog/<<DATA_CATALOG_NAME>>/database/<<DATABASE_NAME>>/table/<<TABLE_NAME>>"
"arn:aws:athena:<<AWS_REGION>>:<<ACCOUNT_ID>>:datacatalog/<<DATA_CATALOG_NAME>>/database/<<DATABASE_NAME>>/table/<<TABLE_NAME>>/column/<<COLUMN_NAME>>"
]
},
{
"Action": [
"lambda:InvokeFunction"
],
"Effect": "Allow",
"Resource": [
"arn:aws:lambda:<<AWS_REGION>>:<<ACCOUNT_ID>>:function:<<CONNECTOR_NAME>>"
]
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:DescribeKey"
],
"Resource": [
"arn:aws:kms:<<AWS_REGION>>:<<ACCOUNT_ID>>:key/<<KMS_KEY_ID>>"
]
}
]
}
Expand Down
2 changes: 1 addition & 1 deletion ...t/v1.5.x/developers/contribute/developing-a-new-connector/define-json-schema.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ Now that you have your Connection defined in the JSON Schema, we can proceed to

{%inlineCallout
color="violet-70"
bold="Develop the Ingesion Code"
bold="Develop the Ingestion Code"
icon="MdArrowForward"
href="/developers/contribute/developing-a-new-connector/develop-ingestion-code"%}
Learn what you need to implement for the Connector's logic
Expand Down
2 changes: 1 addition & 1 deletion openmetadata-docs/content/v1.5.x/releases/releases/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ We are also making the discovery of data more accessible for users introducing a

{% youtube videoId="45ekUIRO1Ec" start="0:00" end="1:11" width="560px" height="315px" /%}

## Pipeline Status Widget
## Pipeline Status Widget (Collate)

We are also adding another widget you can use to customize the Landing Page of the User Personas in your organization.

Expand Down
19 changes: 19 additions & 0 deletions openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/athena/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,25 @@ And is defined as:
"arn:aws:athena:<<AWS_REGION>>:<<ACCOUNT_ID>>:datacatalog/<<DATA_CATALOG_NAME>>/database/<<DATABASE_NAME>>/table/<<TABLE_NAME>>"
"arn:aws:athena:<<AWS_REGION>>:<<ACCOUNT_ID>>:datacatalog/<<DATA_CATALOG_NAME>>/database/<<DATABASE_NAME>>/table/<<TABLE_NAME>>/column/<<COLUMN_NAME>>"
]
},
{
"Action": [
"lambda:InvokeFunction"
],
"Effect": "Allow",
"Resource": [
"arn:aws:lambda:<<AWS_REGION>>:<<ACCOUNT_ID>>:function:<<CONNECTOR_NAME>>"
]
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:DescribeKey"
],
"Resource": [
"arn:aws:kms:<<AWS_REGION>>:<<ACCOUNT_ID>>:key/<<KMS_KEY_ID>>"
]
}
]
}
Expand Down
19 changes: 19 additions & 0 deletions openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/database/athena/yaml.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,25 @@ And is defined as:
"arn:aws:athena:<<AWS_REGION>>:<<ACCOUNT_ID>>:datacatalog/<<DATA_CATALOG_NAME>>/database/<<DATABASE_NAME>>/table/<<TABLE_NAME>>"
"arn:aws:athena:<<AWS_REGION>>:<<ACCOUNT_ID>>:datacatalog/<<DATA_CATALOG_NAME>>/database/<<DATABASE_NAME>>/table/<<TABLE_NAME>>/column/<<COLUMN_NAME>>"
]
},
{
"Action": [
"lambda:InvokeFunction"
],
"Effect": "Allow",
"Resource": [
"arn:aws:lambda:<<AWS_REGION>>:<<ACCOUNT_ID>>:function:<<CONNECTOR_NAME>>"
]
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:DescribeKey"
],
"Resource": [
"arn:aws:kms:<<AWS_REGION>>:<<ACCOUNT_ID>>:key/<<KMS_KEY_ID>>"
]
}
]
}
Expand Down
2 changes: 1 addition & 1 deletion ...SNAPSHOT/developers/contribute/developing-a-new-connector/define-json-schema.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ Now that you have your Connection defined in the JSON Schema, we can proceed to

{%inlineCallout
color="violet-70"
bold="Develop the Ingesion Code"
bold="Develop the Ingestion Code"
icon="MdArrowForward"
href="/developers/contribute/developing-a-new-connector/develop-ingestion-code"%}
Learn what you need to implement for the Connector's logic
Expand Down
2 changes: 1 addition & 1 deletion openmetadata-docs/content/v1.6.x-SNAPSHOT/releases/releases/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ We are also making the discovery of data more accessible for users introducing a

{% youtube videoId="45ekUIRO1Ec" start="0:00" end="1:11" width="560px" height="315px" /%}

## Pipeline Status Widget
## Pipeline Status Widget (Collate)

We are also adding another widget you can use to customize the Landing Page of the User Personas in your organization.

Expand Down
34 changes: 30 additions & 4 deletions openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,10 +391,15 @@ private void validateExtension(
fieldValue.toString(),
customPropertyType,
propertyConfig);
case "enum" -> {
List<String> enumKeys = listOrEmpty(fieldToInternalArray(fieldValue.toString()));
fieldValue = enumKeys.isEmpty() ? null : enumKeys;
}
case "enum" -> fieldValue =
parseEnumType(
printer,
csvRecord,
fieldNumber,
fieldName,
customPropertyType,
fieldValue,
propertyConfig);
case "timeInterval" -> fieldValue =
parseTimeInterval(printer, csvRecord, fieldNumber, fieldName, fieldValue);
case "number", "integer", "timestamp" -> fieldValue =
Expand Down Expand Up @@ -573,6 +578,27 @@ private Object parseTableType(
return tableJson;
}

private Object parseEnumType(
CSVPrinter printer,
CSVRecord csvRecord,
int fieldNumber,
String fieldName,
String customPropertyType,
Object fieldValue,
String propertyConfig)
throws IOException {
List<String> enumKeys = listOrEmpty(fieldToInternalArray(fieldValue.toString()));
try {
EntityRepository.validateEnumKeys(fieldName, JsonUtils.valueToTree(enumKeys), propertyConfig);
} catch (Exception e) {
importFailure(
printer,
invalidCustomPropertyValue(fieldNumber, fieldName, customPropertyType, e.getMessage()),
csvRecord);
}
return enumKeys.isEmpty() ? null : enumKeys;
}

private void validateAndUpdateExtension(
CSVPrinter printer,
CSVRecord csvRecord,
Expand Down
22 changes: 22 additions & 0 deletions openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@
import org.openmetadata.schema.type.api.BulkOperationResult;
import org.openmetadata.schema.type.api.BulkResponse;
import org.openmetadata.schema.type.csv.CsvImportResult;
import org.openmetadata.schema.type.customProperties.EnumConfig;
import org.openmetadata.schema.type.customProperties.TableConfig;
import org.openmetadata.schema.utils.EntityInterfaceUtil;
import org.openmetadata.service.Entity;
Expand Down Expand Up @@ -1526,6 +1527,7 @@ private void validateAndUpdateExtensionBasedOnPropertyType(
jsonNode.put(fieldName, formattedValue);
}
case "table-cp" -> validateTableType(fieldValue, propertyConfig, fieldName);
case "enum" -> validateEnumKeys(fieldName, fieldValue, propertyConfig);
default -> {}
}
}
Expand Down Expand Up @@ -1608,6 +1610,26 @@ private void validateTableType(JsonNode fieldValue, String propertyConfig, Strin
}
}

public static void validateEnumKeys(
String fieldName, JsonNode fieldValue, String propertyConfig) {
JsonNode propertyConfigNode = JsonUtils.readTree(propertyConfig);
EnumConfig config = JsonUtils.treeToValue(propertyConfigNode, EnumConfig.class);

if (!config.getMultiSelect() && fieldValue.size() > 1) {
throw new IllegalArgumentException(
String.format("Only one value allowed for non-multiSelect %s property", fieldName));
}
Set<String> validValues = new HashSet<>(config.getValues());
Set<String> fieldValues = new HashSet<>();
fieldValue.forEach(value -> fieldValues.add(value.asText()));

if (!validValues.containsAll(fieldValues)) {
fieldValues.removeAll(validValues);
throw new IllegalArgumentException(
String.format("Values '%s' not supported for property %s", fieldValues, fieldName));
}
}

public final void storeExtension(EntityInterface entity) {
JsonNode jsonNode = JsonUtils.valueToTree(entity.getExtension());
Iterator<Entry<String, JsonNode>> customFields = jsonNode.fields();
Expand Down
4 changes: 3 additions & 1 deletion ...ervice/src/main/java/org/openmetadata/service/security/AuthenticationCodeFlowHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -916,7 +916,9 @@ private void populateCredentialsFromTokenResponse(
OIDCTokenResponse tokenSuccessResponse, OidcCredentials credentials) {
OIDCTokens oidcTokens = tokenSuccessResponse.getOIDCTokens();
credentials.setAccessToken(oidcTokens.getAccessToken());
credentials.setRefreshToken(oidcTokens.getRefreshToken());
if (oidcTokens.getRefreshToken() != null) {
credentials.setRefreshToken(oidcTokens.getRefreshToken());
}
if (oidcTokens.getIDToken() != null) {
credentials.setIdToken(oidcTokens.getIDToken());
}
Expand Down
Loading

0 comments on commit 529cb5f

Please sign in to comment.