diff --git a/config/loader/aws/databricks.config.reference.hocon b/config/loader/aws/databricks.config.reference.hocon index bf0513eb4..fd0d14527 100644 --- a/config/loader/aws/databricks.config.reference.hocon +++ b/config/loader/aws/databricks.config.reference.hocon @@ -27,7 +27,7 @@ "oauth": { "clientId": "client-id" "clientSecret": ${OAUTH_CLIENT_SECRET} - } + }, # Optional. Override the Databricks default catalog, e.g. with a Unity catalog name. "catalog": "hive_metastore", # DB schema diff --git a/config/loader/azure/databricks.config.reference.hocon b/config/loader/azure/databricks.config.reference.hocon index 8b45a39a6..f7a7381ff 100644 --- a/config/loader/azure/databricks.config.reference.hocon +++ b/config/loader/azure/databricks.config.reference.hocon @@ -43,6 +43,10 @@ "parameterName": "snowplow.databricks.password" } }, + "oauth": { + "clientId": "client-id" + "clientSecret": ${OAUTH_CLIENT_SECRET} + }, # Optional. Override the Databricks default catalog, e.g. with a Unity catalog name. "catalog": "hive_metastore", # DB schema diff --git a/config/loader/gcp/databricks.config.reference.hocon b/config/loader/gcp/databricks.config.reference.hocon index c6fbd4f83..492706161 100644 --- a/config/loader/gcp/databricks.config.reference.hocon +++ b/config/loader/gcp/databricks.config.reference.hocon @@ -21,6 +21,10 @@ "parameterName": "snowplow.databricks.password" } }, + "oauth": { + "clientId": "client-id" + "clientSecret": ${OAUTH_CLIENT_SECRET} + }, # Optional. Override the Databricks default catalog, e.g. with a Unity catalog name. "catalog": "hive_metastore", # DB schema diff --git a/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala b/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala index 0e12906ac..7dd9828c4 100644 --- a/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala +++ b/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/ConfigSpec.scala @@ -112,7 +112,8 @@ class ConfigSpec extends Specification { val result = getConfigFromResource("/loader/aws/databricks.config.minimal.hocon", testParseConfig) val storage = ConfigSpec.exampleStorage.copy( catalog = None, - password = StorageTarget.PasswordConfig.PlainText("Supersecret1") + password = Some(StorageTarget.PasswordConfig.PlainText("Supersecret1")), + oauth = None ) val cloud = Config.Cloud.AWS(RegionSpec.DefaultTestRegion, exampleMessageQueue.copy(region = Some(RegionSpec.DefaultTestRegion))) val retries = exampleRetries.copy(cumulativeBound = Some(20.minutes)) @@ -140,7 +141,8 @@ class ConfigSpec extends Specification { val result = getConfigFromResource("/loader/gcp/databricks.config.minimal.hocon", testParseConfig) val storage = ConfigSpec.exampleStorage.copy( catalog = None, - password = StorageTarget.PasswordConfig.PlainText("Supersecret1") + password = Some(StorageTarget.PasswordConfig.PlainText("Supersecret1")), + oauth = None ) val retries = exampleRetries.copy(cumulativeBound = Some(20.minutes)) val readyCheck = exampleReadyCheck.copy(strategy = Config.Strategy.Constant, backoff = 15.seconds) @@ -167,7 +169,8 @@ class ConfigSpec extends Specification { val result = getConfigFromResource("/loader/azure/databricks.config.minimal.hocon", testParseConfig) val storage = ConfigSpec.exampleStorage.copy( catalog = None, - password = StorageTarget.PasswordConfig.PlainText("Supersecret1") + password = Some(StorageTarget.PasswordConfig.PlainText("Supersecret1")), + oauth = None ) val retries = exampleRetries.copy(cumulativeBound = Some(20.minutes)) val readyCheck = exampleReadyCheck.copy(strategy = Config.Strategy.Constant, backoff = 15.seconds) @@ -200,7 +203,8 @@ object ConfigSpec { "atomic", 443, "/databricks/http/path", - StorageTarget.PasswordConfig.EncryptedKey(StorageTarget.EncryptedConfig("snowplow.databricks.password")), + Some(StorageTarget.PasswordConfig.EncryptedKey(StorageTarget.EncryptedConfig("snowplow.databricks.password"))), + Some(StorageTarget.Databricks.OAuth("client-id", "client-secret")), None, "snowplow-rdbloader-oss", StorageTarget.LoadAuthMethod.NoCreds, diff --git a/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/DatabricksSpec.scala b/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/DatabricksSpec.scala index b0ef7ddfa..8d592a27f 100644 --- a/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/DatabricksSpec.scala +++ b/modules/databricks-loader/src/test/scala/com/snowplowanalytics/snowplow/loader/databricks/DatabricksSpec.scala @@ -227,7 +227,8 @@ object DatabricksSpec { "snowplow", 443, "some/path", - StorageTarget.PasswordConfig.PlainText("xxx"), + Some(StorageTarget.PasswordConfig.PlainText("xxx")), + None, None, "useragent", StorageTarget.LoadAuthMethod.NoCreds, diff --git a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfigSpec.scala b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfigSpec.scala index 644584670..ffdd4685f 100644 --- a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfigSpec.scala +++ b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/CliConfigSpec.scala @@ -90,7 +90,7 @@ class CliConfigSpec extends Specification { val result = CliConfig.parse[IO](cli).value.unsafeRunSync() result must beRight.like { case CliConfig(config, _, resolverConfig) => - config.storage.password.getUnencrypted must beEqualTo("Supersecret password from substitution!") + config.storage.credentials.get.password.getUnencrypted must beEqualTo("Supersecret password from substitution!") resolverConfig must beEqualTo(expectedResolver) } diff --git a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/StorageTargetSpec.scala b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/StorageTargetSpec.scala index 4e78db106..07e09802e 100644 --- a/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/StorageTargetSpec.scala +++ b/modules/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/StorageTargetSpec.scala @@ -162,6 +162,10 @@ class StorageTargetSpec extends Specification { "port": 443, "httpPath": "http/path", "password": "Supersecret1", + "oauth": { + "clientId": "client-id", + "clientSecret": "client-secret" + }, "userAgent": "snowplow-rdbloader-oss", "eventsOptimizePeriod": "2 days", "loadAuthMethod": { @@ -177,7 +181,8 @@ class StorageTargetSpec extends Specification { schema = "snowplow", port = 443, httpPath = "http/path", - password = StorageTarget.PasswordConfig.PlainText("Supersecret1"), + password = Some(StorageTarget.PasswordConfig.PlainText("Supersecret1")), + oauth = Some(StorageTarget.Databricks.OAuth("client-id", "client-secret")), sshTunnel = None, userAgent = "snowplow-rdbloader-oss", loadAuthMethod = StorageTarget.LoadAuthMethod.NoCreds, diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index 37331e0f8..081a131ed 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -243,7 +243,13 @@ object BuildSettings { Docker / packageName := "rdb-loader-databricks", initialCommands := "import com.snowplowanalytics.snowplow.loader.databricks._", Compile / mainClass := Some("com.snowplowanalytics.snowplow.loader.databricks.Main"), - Compile / unmanagedJars += file("DatabricksJDBC42.jar") + Compile / unmanagedJars += file("DatabricksJDBC42.jar"), + // used in extended configuration parsing unit tests + Test / envVars := Map( + "OAUTH_CLIENT_SECRET" -> "client-secret" + ), + // envVars works only when fork is enabled + Test / fork := true ) ++ buildSettings ++ addExampleConfToTestCp ++ assemblySettings ++ dynVerSettings lazy val transformerBatchBuildSettings =