From 9809e0ee4c33d936273ec082ed603437a855729a Mon Sep 17 00:00:00 2001 From: Michael Collado <40346148+collado-mike@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:39:17 -0800 Subject: [PATCH] Working impl of HK2 dependency injection (#493) * Working impl of HK2 dependency injection * Add RealmScope and make EntityCache and PolarisMetaStoreManager realm-scoped services * Remove dependency on HK2 from core module * Fix EntityCache to work with RealmScope * fix TimedApplicationEventListener.class * Add integration test to validate EntityCache in RealmScope * Added comments on PolarisApplication startup * Update HK2 setup to use PolarisApplicationConfig as interface to serviceLocator and handle DI for configured instances * Fixed RealmEntityManagerFactory to use EntityCache provider * Update Dockerfile to use entrypoint and fix vended-credentials delegation header * fix spotless * Update polaris-core/src/main/java/org/apache/polaris/core/context/RealmScope.java Co-authored-by: Robert Stupp * Removed Jackson type annotations from interfaces * Address PR comments * Update build versions to include jakarta.inject * Removed unnecessary Discoverable service files * Fixes for rebase on main * Update Jersey ServiceLocator to use bootstrapped locator as parent * Addressed more PR comments * Update RealmScopeContext to use request-scoped RealmContext bean * Update CDI impls to use @Identifier and replaced all setter injection with field injection * Fix helm test to match expectation * Revert PolarisStorageIntegrationProviderImpl back to service module * Fix missing project mentions in LICENSE * Fix lint issue in helm values * Add logic in build to merge multiple hks-locator files --------- Co-authored-by: Robert Stupp Co-authored-by: Michael Collado --- LICENSE | 9 + .../persistence/eclipselink/build.gradle.kts | 2 + ...pseLinkPolarisMetaStoreManagerFactory.java | 11 +- .../resources/META-INF/hk2-locator/default | 9 +- .../io.dropwizard.jackson.Discoverable | 20 - gradle/libs.versions.toml | 2 + helm/polaris/tests/configmap_test.yaml | 18 +- helm/polaris/values.yaml | 6 +- polaris-core/build.gradle.kts | 2 + .../core/auth/PolarisAuthorizerImpl.java | 2 + .../polaris/core/context/RealmScoped.java | 18 +- .../core/monitor/PolarisMetricRegistry.java | 2 + .../LocalPolarisMetaStoreManagerFactory.java | 14 - .../persistence/MetaStoreManagerFactory.java | 8 - .../persistence/PolarisEntityManager.java | 7 +- .../core/persistence/cache/EntityCache.java | 4 + polaris-server.yml | 18 +- polaris-service/build.gradle.kts | 2 + .../service/BootstrapRealmsCommand.java | 16 +- .../polaris/service/PolarisApplication.java | 393 +++++++++++------ .../polaris/service/PurgeRealmsCommand.java | 16 +- .../TimedApplicationEventListener.java | 2 + .../service/admin/PolarisServiceImpl.java | 2 + .../auth/BasePolarisAuthenticator.java | 11 +- .../service/auth/DefaultOAuth2ApiService.java | 24 +- .../auth/DefaultPolarisAuthenticator.java | 22 +- .../auth/DiscoverableAuthenticator.java | 6 +- .../service/auth/JWTRSAKeyPairFactory.java | 16 +- .../service/auth/JWTSymmetricKeyFactory.java | 11 +- .../polaris/service/auth/KeyProvider.java | 5 +- ...InlineBearerTokenPolarisAuthenticator.java | 2 + .../service/auth/TestOAuth2ApiService.java | 18 +- .../service/auth/TokenBrokerFactory.java | 5 +- .../catalog/IcebergCatalogAdapter.java | 2 + .../catalog/io/DefaultFileIOFactory.java | 4 +- .../service/catalog/io/FileIOFactory.java | 3 +- .../io/WasbTranslatingFileIOFactory.java | 4 +- .../service/config/OAuth2ApiService.java | 6 +- .../config/PolarisApplicationConfig.java | 227 +++++++++- .../config/RealmEntityManagerFactory.java | 17 +- .../service/context/CallContextResolver.java | 6 +- .../context/DefaultContextResolver.java | 28 +- .../PolarisCallContextCatalogFactory.java | 2 + .../service/context/RealmContextResolver.java | 6 +- .../service/context/RealmScopeContext.java | 99 +++++ ...nMemoryPolarisMetaStoreManagerFactory.java | 18 +- .../persistence/cache/EntityCacheFactory.java | 47 ++ .../service/ratelimiter/NoOpRateLimiter.java | 4 +- .../service/ratelimiter/RateLimiter.java | 6 +- .../ratelimiter/RateLimiterFilter.java | 2 + .../RealmTokenBucketRateLimiter.java | 4 +- .../ratelimiter/TokenBucketRateLimiter.java | 2 + ...PolarisStorageIntegrationProviderImpl.java | 2 + .../service/tracing/TracingFilter.java | 2 + .../resources/META-INF/hk2-locator/default | 88 ++++ .../io.dropwizard.jackson.Discoverable | 27 -- ...he.polaris.service.auth.TokenBrokerFactory | 21 - ...e.polaris.service.catalog.io.FileIOFactory | 21 - ...he.polaris.service.config.OAuth2ApiService | 21 - ...olaris.service.context.CallContextResolver | 20 - ...laris.service.context.RealmContextResolver | 20 - ...he.polaris.service.ratelimiter.RateLimiter | 21 - .../PolarisApplicationConfigurationTest.java | 5 +- .../service/admin/PolarisAuthzTestBase.java | 5 +- .../admin/PolarisOverlappingTableTest.java | 3 +- .../admin/PolarisRealmEntityCacheTest.java | 278 ++++++++++++ .../catalog/BasePolarisCatalogTest.java | 13 +- .../catalog/BasePolarisCatalogViewTest.java | 4 +- .../catalog/io/FileIOIntegrationTest.java | 2 +- .../service/catalog/io/TestFileIOFactory.java | 4 +- .../MockRealmTokenBucketRateLimiter.java | 4 +- .../test/PolarisConnectionExtension.java | 8 +- .../resources/META-INF/hk2-locator/default} | 15 +- ...ris.service.auth.DiscoverableAuthenticator | 20 - ...e.polaris.service.catalog.io.FileIOFactory | 20 - ...he.polaris.service.ratelimiter.RateLimiter | 20 - .../polaris-server-integrationtest.yml | 9 +- regtests/Dockerfile | 2 +- .../src/test_spark_sql_s3_with_privileges.py | 401 ++++++++++-------- server-templates/api.mustache | 3 + 80 files changed, 1427 insertions(+), 822 deletions(-) rename polaris-core/src/main/java/org/apache/polaris/core/monitor/MetricRegistryAware.java => extension/persistence/eclipselink/src/main/resources/META-INF/hk2-locator/default (76%) delete mode 100644 extension/persistence/eclipselink/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable rename polaris-service/src/main/java/org/apache/polaris/service/config/ConfigurationStoreAware.java => polaris-core/src/main/java/org/apache/polaris/core/context/RealmScoped.java (67%) create mode 100644 polaris-service/src/main/java/org/apache/polaris/service/context/RealmScopeContext.java create mode 100644 polaris-service/src/main/java/org/apache/polaris/service/persistence/cache/EntityCacheFactory.java create mode 100644 polaris-service/src/main/resources/META-INF/hk2-locator/default delete mode 100644 polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.auth.TokenBrokerFactory delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.config.OAuth2ApiService delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.CallContextResolver delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.RealmContextResolver delete mode 100644 polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter create mode 100644 polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisRealmEntityCacheTest.java rename polaris-service/src/{main/java/org/apache/polaris/service/tracing/OpenTelemetryAware.java => test/resources/META-INF/hk2-locator/default} (66%) delete mode 100644 polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.auth.DiscoverableAuthenticator delete mode 100644 polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory delete mode 100644 polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter diff --git a/LICENSE b/LICENSE index 2879d70d5..406856eec 100644 --- a/LICENSE +++ b/LICENSE @@ -375,6 +375,14 @@ io.prometheus:prometheus-metrics-exposition-formats io.prometheus:prometheus-metrics-model io.prometheus:prometheus-metrics-shaded-protobuf io.prometheus:prometheus-metrics-tracer-common +io.smallrye.common:smallrye-common-annotation +io.smallrye.common:smallrye-common-classloader +io.smallrye.common:smallrye-common-constraint +io.smallrye.common:smallrye-common-expression +io.smallrye.common:smallrye-common-function +io.smallrye.config:smallrye-config +io.smallrye.config:smallrye-config-common +io.smallrye.config:smallrye-config-core io.swagger:swagger-annotations io.swagger:swagger-core io.swagger:swagger-jaxrs @@ -446,6 +454,7 @@ org.eclipse.jetty:jetty-servlets org.eclipse.jetty:jetty-util org.eclipse.jetty:jetty-webapp org.eclipse.jetty:jetty-xml +org.eclipse.microprofile.config:microprofile-config-api org.glassfish.jersey.containers:jersey-container-servlet org.glassfish.jersey.containers:jersey-container-servlet-core org.glassfish.jersey.core:jersey-client diff --git a/extension/persistence/eclipselink/build.gradle.kts b/extension/persistence/eclipselink/build.gradle.kts index 4de0833ac..af37bac9d 100644 --- a/extension/persistence/eclipselink/build.gradle.kts +++ b/extension/persistence/eclipselink/build.gradle.kts @@ -32,6 +32,8 @@ dependencies { implementation(project(":polaris-jpa-model")) implementation(libs.eclipselink) implementation(platform(libs.dropwizard.bom)) + implementation(libs.jakarta.inject.api) + implementation(libs.smallrye) implementation("io.dropwizard:dropwizard-jackson") val eclipseLinkDeps: String? = project.findProperty("eclipseLinkDeps") as String? eclipseLinkDeps?.let { diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java index 03c7afabb..21e466b78 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java @@ -19,29 +19,32 @@ package org.apache.polaris.extension.persistence.impl.eclipselink; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import io.dropwizard.jackson.Discoverable; +import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nonnull; +import jakarta.inject.Inject; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; /** * The implementation of Configuration interface for configuring the {@link PolarisMetaStoreManager} * using an EclipseLink based meta store to store and retrieve all Polaris metadata. It can be * configured through persistence.xml to use supported RDBMS as the meta store. */ -@JsonTypeName("eclipse-link") +@Identifier("eclipse-link") public class EclipseLinkPolarisMetaStoreManagerFactory - extends LocalPolarisMetaStoreManagerFactory implements Discoverable { + extends LocalPolarisMetaStoreManagerFactory { @JsonProperty("conf-file") private String confFile; @JsonProperty("persistence-unit") private String persistenceUnitName; + @Inject protected PolarisStorageIntegrationProvider storageIntegration; + @Override protected PolarisEclipseLinkStore createBackingStore(@Nonnull PolarisDiagnostics diagnostics) { return new PolarisEclipseLinkStore(diagnostics); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/monitor/MetricRegistryAware.java b/extension/persistence/eclipselink/src/main/resources/META-INF/hk2-locator/default similarity index 76% rename from polaris-core/src/main/java/org/apache/polaris/core/monitor/MetricRegistryAware.java rename to extension/persistence/eclipselink/src/main/resources/META-INF/hk2-locator/default index 2977d60a5..ccebc9f69 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/monitor/MetricRegistryAware.java +++ b/extension/persistence/eclipselink/src/main/resources/META-INF/hk2-locator/default @@ -16,9 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.monitor; +[org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory]S +contract={org.apache.polaris.core.persistence.MetaStoreManagerFactory} +name=eclipse-link +qualifier={io.smallrye.common.annotation.Identifier} -/** Allows setting a configured instance of {@link PolarisMetricRegistry} */ -public interface MetricRegistryAware { - void setMetricRegistry(PolarisMetricRegistry metricRegistry); -} diff --git a/extension/persistence/eclipselink/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable b/extension/persistence/eclipselink/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable deleted file mode 100644 index 58f8891eb..000000000 --- a/extension/persistence/eclipselink/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5fd5fe49a..543b36cda 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -52,6 +52,7 @@ hadoop-hdfs-client = { module = "org.apache.hadoop:hadoop-hdfs-client", version. iceberg-bom = { module = "org.apache.iceberg:iceberg-bom", version.ref = "iceberg" } jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version = "2.17.2" } jakarta-annotation-api = { module = "jakarta.annotation:jakarta.annotation-api", version = "3.0.0" } +jakarta-inject-api = { module = "jakarta.inject:jakarta.inject-api", version = "2.0.1" } jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api", version = "3.1.0" } jakarta-persistence-api = { module = "jakarta.persistence:jakarta.persistence-api", version = "3.1.0" } javax-annotation-api = { module = "javax.annotation:javax.annotation-api", version = "1.3.2" } @@ -71,6 +72,7 @@ swagger-annotations = { module = "io.swagger:swagger-annotations", version.ref = swagger-jaxrs = { module = "io.swagger:swagger-jaxrs", version.ref = "swagger" } testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version = "1.20.3" } threeten-extra = { module = "org.threeten:threeten-extra", version = "1.8.0" } +smallrye = { module = "io.smallrye.config:smallrye-config", version = "3.10.2" } [plugins] openapi-generator = { id = "org.openapi.generator", version = "7.6.0" } diff --git a/helm/polaris/tests/configmap_test.yaml b/helm/polaris/tests/configmap_test.yaml index b9b5099a2..859ae1992 100644 --- a/helm/polaris/tests/configmap_test.yaml +++ b/helm/polaris/tests/configmap_test.yaml @@ -184,14 +184,11 @@ tests: factoryType: default oauth2: type: default - tokenBroker: - type: symmetric-key - secret: polaris + tokenBroker: + type: symmetric-key + secret: polaris authenticator: class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator - tokenBroker: - type: symmetric-key - secret: polaris cors: allowed-origins: - http://localhost:8080 @@ -228,9 +225,6 @@ tests: polaris-server.yml: |- authenticator: class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator - tokenBroker: - secret: polaris - type: symmetric-key callContextResolver: type: default cors: @@ -274,9 +268,6 @@ tests: persistence-unit: polaris type: eclipse-link oauth2: - tokenBroker: - secret: polaris - type: symmetric-key type: default rateLimiter: type: no-op @@ -294,3 +285,6 @@ tests: requestLog: appenders: - type: console + tokenBroker: + secret: polaris + type: symmetric-key diff --git a/helm/polaris/values.yaml b/helm/polaris/values.yaml index 9ddce06de..0dd9f969c 100644 --- a/helm/polaris/values.yaml +++ b/helm/polaris/values.yaml @@ -321,9 +321,9 @@ polarisServerConfig: oauth2: type: test # type: default # - uncomment to support Auth0 JWT tokens - # tokenBroker: - # type: symmetric-key - # secret: polaris +# tokenBroker: +# type: symmetric-key +# secret: polaris authenticator: class: org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator diff --git a/polaris-core/build.gradle.kts b/polaris-core/build.gradle.kts index 659450764..f65dc0355 100644 --- a/polaris-core/build.gradle.kts +++ b/polaris-core/build.gradle.kts @@ -67,7 +67,9 @@ dependencies { implementation(libs.javax.inject) implementation(libs.swagger.annotations) implementation(libs.swagger.jaxrs) + implementation(libs.jakarta.inject.api) implementation(libs.jakarta.validation.api) + implementation(libs.smallrye) implementation("org.apache.iceberg:iceberg-aws") implementation(platform(libs.awssdk.bom)) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java index 7b6e91e46..96f449acb 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java @@ -93,6 +93,7 @@ import com.google.common.collect.SetMultimap; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import jakarta.inject.Inject; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -460,6 +461,7 @@ public class PolarisAuthorizerImpl implements PolarisAuthorizer { private final PolarisConfigurationStore featureConfig; + @Inject public PolarisAuthorizerImpl(PolarisConfigurationStore featureConfig) { this.featureConfig = featureConfig; } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/ConfigurationStoreAware.java b/polaris-core/src/main/java/org/apache/polaris/core/context/RealmScoped.java similarity index 67% rename from polaris-service/src/main/java/org/apache/polaris/service/config/ConfigurationStoreAware.java rename to polaris-core/src/main/java/org/apache/polaris/core/context/RealmScoped.java index 645f1b001..9bf456f58 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/ConfigurationStoreAware.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/context/RealmScoped.java @@ -16,12 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.config; +package org.apache.polaris.core.context; -import org.apache.polaris.core.PolarisConfigurationStore; +import jakarta.inject.Scope; +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; -/** Interface allows injection of a {@link PolarisConfigurationStore} */ -public interface ConfigurationStoreAware { - - void setConfigurationStore(PolarisConfigurationStore configurationStore); -} +@Scope +@Documented +@Retention(java.lang.annotation.RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.METHOD}) +public @interface RealmScoped {} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/monitor/PolarisMetricRegistry.java b/polaris-core/src/main/java/org/apache/polaris/core/monitor/PolarisMetricRegistry.java index e8f20d9cb..ea2734722 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/monitor/PolarisMetricRegistry.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/monitor/PolarisMetricRegistry.java @@ -28,6 +28,7 @@ import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; import io.micrometer.core.instrument.binder.system.ProcessorMetrics; +import jakarta.inject.Inject; import java.lang.reflect.Method; import java.util.Collections; import java.util.concurrent.ConcurrentHashMap; @@ -68,6 +69,7 @@ public class PolarisMetricRegistry { public static final String SUFFIX_ERROR = ".error"; public static final String SUFFIX_REALM = ".realm"; + @Inject public PolarisMetricRegistry(MeterRegistry meterRegistry) { this.meterRegistry = meterRegistry; new ClassLoaderMetrics().bindTo(meterRegistry); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java index 382a557ba..25bea896f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java @@ -34,8 +34,6 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.monitor.PolarisMetricRegistry; -import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,8 +52,6 @@ public abstract class LocalPolarisMetaStoreManagerFactory final Map> sessionSupplierMap = new HashMap<>(); protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - protected PolarisStorageIntegrationProvider storageIntegration; - private static final Logger LOGGER = LoggerFactory.getLogger(LocalPolarisMetaStoreManagerFactory.class); @@ -157,16 +153,6 @@ public synchronized StorageCredentialCache getOrCreateStorageCredentialCache( return storageCredentialCacheMap.get(realmContext.getRealmIdentifier()); } - @Override - public void setMetricRegistry(PolarisMetricRegistry metricRegistry) { - // no-op - } - - @Override - public void setStorageIntegrationProvider(PolarisStorageIntegrationProvider storageIntegration) { - this.storageIntegration = storageIntegration; - } - /** * This method bootstraps service for a given realm: i.e. creates all the needed entities in the * metastore and creates a root service principal. After that we rotate the root principal diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java index 4352a975b..7cdb36c5a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java @@ -18,21 +18,17 @@ */ package org.apache.polaris.core.persistence; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.util.List; import java.util.Map; import java.util.function.Supplier; import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.monitor.PolarisMetricRegistry; -import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.cache.StorageCredentialCache; /** * Configuration interface for configuring the {@link PolarisMetaStoreManager} via Dropwizard * configuration */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") public interface MetaStoreManagerFactory { PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext); @@ -41,10 +37,6 @@ public interface MetaStoreManagerFactory { StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext); - void setStorageIntegrationProvider(PolarisStorageIntegrationProvider storageIntegrationProvider); - - void setMetricRegistry(PolarisMetricRegistry metricRegistry); - Map bootstrapRealms(List realms); /** Purge all metadata for the realms provided */ diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityManager.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityManager.java index ba7398aaa..3d0d2457a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityManager.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisEntityManager.java @@ -51,11 +51,14 @@ public class PolarisEntityManager { /** * @param metaStoreManager the metastore manager for the current realm * @param credentialCache the storage credential cache for the current realm + * @param entityCache the entity cache */ public PolarisEntityManager( - PolarisMetaStoreManager metaStoreManager, StorageCredentialCache credentialCache) { + PolarisMetaStoreManager metaStoreManager, + StorageCredentialCache credentialCache, + EntityCache entityCache) { this.metaStoreManager = metaStoreManager; - this.entityCache = new EntityCache(metaStoreManager); + this.entityCache = entityCache; this.credentialCache = credentialCache; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java index 05efba7fb..8836a171c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java @@ -23,17 +23,20 @@ import com.github.benmanes.caffeine.cache.RemovalListener; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import jakarta.inject.Inject; import java.util.AbstractMap; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.context.RealmScoped; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.persistence.cache.PolarisRemoteCache.CachedEntryResult; /** The entity cache, can be private or shared */ +@RealmScoped public class EntityCache { // cache mode @@ -53,6 +56,7 @@ public class EntityCache { * * @param polarisRemoteCache the meta store manager implementation */ + @Inject public EntityCache(@Nonnull PolarisRemoteCache polarisRemoteCache) { // by name cache diff --git a/polaris-server.yml b/polaris-server.yml index e147d4d48..e88307bf6 100644 --- a/polaris-server.yml +++ b/polaris-server.yml @@ -93,17 +93,19 @@ io: # TODO - avoid duplicating token broker config oauth2: type: test -# type: default # - uncomment to support Auth0 JWT tokens -# tokenBroker: -# type: symmetric-key -# secret: polaris authenticator: class: org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator + +# - uncomment to support Auth0 JWT tokens +#oauth2: +# type: default +#authenticator: # class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator # - uncomment to support Auth0 JWT tokens -# tokenBroker: -# type: symmetric-key -# secret: polaris +# +#tokenBroker: +# type: symmetric-key +# secret: polaris cors: allowed-origins: @@ -133,7 +135,7 @@ logging: # Logger-specific levels. loggers: org.apache.iceberg.rest: DEBUG - org.apache.polaris: DEBUG + org.apache.polaris: INFO appenders: diff --git a/polaris-service/build.gradle.kts b/polaris-service/build.gradle.kts index fc7ff6986..ed88b43c3 100644 --- a/polaris-service/build.gradle.kts +++ b/polaris-service/build.gradle.kts @@ -43,6 +43,7 @@ dependencies { exclude("org.apache.zookeeper", "zookeeper") } implementation(libs.hadoop.hdfs.client) + implementation(libs.smallrye) implementation(platform(libs.dropwizard.bom)) implementation("io.dropwizard:dropwizard-core") @@ -251,6 +252,7 @@ val shadowJar = tasks.named("shadowJar") { manifest { attributes["Main-Class"] = "org.apache.polaris.service.PolarisApplication" } mergeServiceFiles() + append("META-INF/hk2-locator/default") isZip64 = true finalizedBy("startScripts") } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java b/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java index 3f282b042..338e687c7 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java @@ -25,9 +25,7 @@ import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.ConfigurationStoreAware; import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.context.CallContextResolver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,17 +46,11 @@ protected void run( Bootstrap bootstrap, Namespace namespace, PolarisApplicationConfig configuration) { - MetaStoreManagerFactory metaStoreManagerFactory = configuration.getMetaStoreManagerFactory(); + MetaStoreManagerFactory metaStoreManagerFactory = + configuration.findService(MetaStoreManagerFactory.class); - PolarisConfigurationStore configurationStore = configuration.getConfigurationStore(); - if (metaStoreManagerFactory instanceof ConfigurationStoreAware) { - ((ConfigurationStoreAware) metaStoreManagerFactory).setConfigurationStore(configurationStore); - } - CallContextResolver callContextResolver = configuration.getCallContextResolver(); - callContextResolver.setMetaStoreManagerFactory(metaStoreManagerFactory); - if (callContextResolver instanceof ConfigurationStoreAware csa) { - csa.setConfigurationStore(configurationStore); - } + PolarisConfigurationStore configurationStore = + configuration.findService(PolarisConfigurationStore.class); // Execute the bootstrap Map results = diff --git a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java index 89287a3e8..be55259ca 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/PolarisApplication.java @@ -25,19 +25,27 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.deser.std.StdValueInstantiator; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.module.SimpleValueInstantiators; +import com.fasterxml.jackson.databind.type.TypeFactory; import io.dropwizard.auth.AuthDynamicFeature; import io.dropwizard.auth.AuthFilter; +import io.dropwizard.auth.Authenticator; import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter; import io.dropwizard.configuration.EnvironmentVariableSubstitutor; import io.dropwizard.configuration.SubstitutingSourceProvider; import io.dropwizard.core.Application; import io.dropwizard.core.setup.Bootstrap; import io.dropwizard.core.setup.Environment; -import io.dropwizard.jackson.Discoverable; -import io.dropwizard.jackson.DiscoverableSubtypeResolver; +import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.prometheusmetrics.PrometheusConfig; import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; import io.opentelemetry.api.OpenTelemetry; @@ -52,6 +60,9 @@ import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import io.opentelemetry.semconv.ServiceAttributes; import io.prometheus.metrics.exporter.servlet.jakarta.PrometheusMetricsServlet; +import jakarta.inject.Inject; +import jakarta.inject.Provider; +import jakarta.inject.Singleton; import jakarta.servlet.DispatcherType; import jakarta.servlet.Filter; import jakarta.servlet.FilterChain; @@ -65,7 +76,6 @@ import java.net.URL; import java.util.Collections; import java.util.EnumSet; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.Executors; @@ -73,28 +83,32 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.iceberg.rest.RESTSerializers; -import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.auth.PolarisAuthorizer; import org.apache.polaris.core.auth.PolarisAuthorizerImpl; +import org.apache.polaris.core.auth.PolarisGrantManager; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.monitor.MetricRegistryAware; +import org.apache.polaris.core.context.RealmScoped; import org.apache.polaris.core.monitor.PolarisMetricRegistry; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.cache.PolarisRemoteCache; import org.apache.polaris.service.admin.PolarisServiceImpl; import org.apache.polaris.service.admin.api.PolarisCatalogsApi; +import org.apache.polaris.service.admin.api.PolarisCatalogsApiService; import org.apache.polaris.service.admin.api.PolarisPrincipalRolesApi; +import org.apache.polaris.service.admin.api.PolarisPrincipalRolesApiService; import org.apache.polaris.service.admin.api.PolarisPrincipalsApi; -import org.apache.polaris.service.auth.DiscoverableAuthenticator; +import org.apache.polaris.service.admin.api.PolarisPrincipalsApiService; import org.apache.polaris.service.catalog.IcebergCatalogAdapter; import org.apache.polaris.service.catalog.api.IcebergRestCatalogApi; +import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService; import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApi; +import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApiService; import org.apache.polaris.service.catalog.api.IcebergRestOAuth2Api; import org.apache.polaris.service.catalog.io.FileIOFactory; -import org.apache.polaris.service.config.ConfigurationStoreAware; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; -import org.apache.polaris.service.config.OAuth2ApiService; import org.apache.polaris.service.config.PolarisApplicationConfig; import org.apache.polaris.service.config.RealmEntityManagerFactory; import org.apache.polaris.service.config.Serializers; @@ -103,30 +117,37 @@ import org.apache.polaris.service.context.CallContextResolver; import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; import org.apache.polaris.service.context.RealmContextResolver; +import org.apache.polaris.service.context.RealmScopeContext; import org.apache.polaris.service.exception.IcebergExceptionMapper; import org.apache.polaris.service.exception.IcebergJerseyViolationExceptionMapper; import org.apache.polaris.service.exception.IcebergJsonProcessingExceptionMapper; import org.apache.polaris.service.exception.PolarisExceptionMapper; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.service.persistence.cache.EntityCacheFactory; import org.apache.polaris.service.ratelimiter.RateLimiterFilter; -import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; import org.apache.polaris.service.task.ManifestFileCleanupTaskHandler; import org.apache.polaris.service.task.TableCleanupTaskHandler; +import org.apache.polaris.service.task.TaskExecutor; import org.apache.polaris.service.task.TaskExecutorImpl; import org.apache.polaris.service.task.TaskFileIOSupplier; import org.apache.polaris.service.throttling.StreamReadConstraintsExceptionMapper; -import org.apache.polaris.service.tracing.OpenTelemetryAware; import org.apache.polaris.service.tracing.TracingFilter; import org.eclipse.jetty.servlets.CrossOriginFilter; +import org.glassfish.hk2.api.Context; +import org.glassfish.hk2.api.Factory; +import org.glassfish.hk2.api.ServiceLocator; +import org.glassfish.hk2.api.TypeLiteral; +import org.glassfish.hk2.utilities.ServiceLocatorUtilities; +import org.glassfish.hk2.utilities.binding.AbstractBinder; +import org.glassfish.jersey.process.internal.RequestScoped; +import org.glassfish.jersey.servlet.ServletProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.services.sts.StsClient; -import software.amazon.awssdk.services.sts.StsClientBuilder; public class PolarisApplication extends Application { private static final Logger LOGGER = LoggerFactory.getLogger(PolarisApplication.class); + private ServiceLocator serviceLocator; public static void main(final String[] args) throws Exception { new PolarisApplication().run(args); @@ -145,7 +166,6 @@ private static void printAsciiArt() throws IOException { @Override public void initialize(Bootstrap bootstrap) { - registerTypes(bootstrap.getObjectMapper()); // Enable variable substitution with environment variables EnvironmentVariableSubstitutor substitutor = new EnvironmentVariableSubstitutor(false); SubstitutingSourceProvider provider = @@ -154,99 +174,189 @@ public void initialize(Bootstrap bootstrap) { bootstrap.addCommand(new BootstrapRealmsCommand()); bootstrap.addCommand(new PurgeRealmsCommand()); + serviceLocator = ServiceLocatorUtilities.createAndPopulateServiceLocator(); + ObjectMapper objectMapper = bootstrap.getObjectMapper(); + + // Register the PolarisApplicationConfig class with the ServiceLocator so that we can inject + // instances of the configuration or certain of the configuration fields into other classes. + // The configuration's postConstruct method registers all the configured fields as factories + // so that they are used for DI. + ServiceLocatorUtilities.addClasses(serviceLocator, PolarisApplicationConfig.class); + TypeFactory typeFactory = TypeFactory.defaultInstance(); + SimpleValueInstantiators instantiators = new SimpleValueInstantiators(); + instantiators.addValueInstantiator( + PolarisApplicationConfig.class, + new ServiceLocatorValueInstantiator( + objectMapper.getDeserializationConfig(), + typeFactory.constructType(PolarisApplicationConfig.class), + serviceLocator)); + + // Use the default ServiceLocator to discover the implementations of various contract providers + // and register them as subtypes with the ObjectMapper. This allows Jackson to discover the + // various implementations and use the type annotations to determine which instance to use when + // parsing the YAML configuration. + SimpleModule module = new SimpleModule(); + serviceLocator + .getDescriptors((c) -> true) + .forEach( + descriptor -> { + try { + Class klazz = + PolarisApplication.class + .getClassLoader() + .loadClass(descriptor.getImplementation()); + String name = descriptor.getName(); + if (name == null) { + module.registerSubtypes(klazz); + } else { + module.registerSubtypes(new NamedType(klazz, name)); + } + } catch (ClassNotFoundException e) { + LOGGER.error("Error loading class {}", descriptor.getImplementation(), e); + throw new RuntimeException("Unable to start Polaris application"); + } + }); + + ServiceLocatorUtilities.addClasses(serviceLocator, PolarisMetricRegistry.class); + ServiceLocatorUtilities.bind( + serviceLocator, + new AbstractBinder() { + @Override + protected void configure() { + bind(setupTracing()).to(OpenTelemetry.class); + bind(new PrometheusMeterRegistry(PrometheusConfig.DEFAULT)).to(MeterRegistry.class); + } + }); + module.setValueInstantiators(instantiators); + objectMapper.registerModule(module); } - private void registerTypes(ObjectMapper mapper) { - // Reuse the DW service discovery method, but unlike the constructor of - // DiscoverableSubtypeResolver, use the first level classes from the `Discoverable` - // service descriptor and register them with the ObjectMapper. - class Discoverer extends DiscoverableSubtypeResolver { - List> discover() { - return discoverServices(Discoverable.class); - } + /** + * Value instantiator that uses the ServiceLocator to create instances of the various service + * types + */ + private static class ServiceLocatorValueInstantiator extends StdValueInstantiator { + private final ServiceLocator serviceLocator; + + public ServiceLocatorValueInstantiator( + DeserializationConfig config, JavaType valueType, ServiceLocator serviceLocator) { + super(config, valueType); + this.serviceLocator = serviceLocator; + } + + @Override + public boolean canCreateUsingDefault() { + return true; + } + + @Override + public boolean canInstantiate() { + return true; + } + + @Override + public Object createUsingDefault(DeserializationContext ctxt) throws IOException { + return ServiceLocatorUtilities.findOrCreateService(serviceLocator, getValueClass()); } - new Discoverer().discover().forEach(mapper::registerSubtypes); } @Override public void run(PolarisApplicationConfig configuration, Environment environment) { - MetaStoreManagerFactory metaStoreManagerFactory = configuration.getMetaStoreManagerFactory(); - - metaStoreManagerFactory.setStorageIntegrationProvider( - new PolarisStorageIntegrationProviderImpl( - () -> { - StsClientBuilder stsClientBuilder = StsClient.builder(); - AwsCredentialsProvider awsCredentialsProvider = configuration.credentialsProvider(); - if (awsCredentialsProvider != null) { - stsClientBuilder.credentialsProvider(awsCredentialsProvider); - } - return stsClientBuilder.build(); - }, - configuration.getGcpCredentialsProvider())); - PolarisMetricRegistry polarisMetricRegistry = - new PolarisMetricRegistry(new PrometheusMeterRegistry(PrometheusConfig.DEFAULT)); - metaStoreManagerFactory.setMetricRegistry(polarisMetricRegistry); + configuration.findService(PolarisMetricRegistry.class); - OpenTelemetry openTelemetry = setupTracing(); - if (metaStoreManagerFactory instanceof OpenTelemetryAware otAware) { - otAware.setOpenTelemetry(openTelemetry); - } - PolarisConfigurationStore configurationStore = configuration.getConfigurationStore(); - if (metaStoreManagerFactory instanceof ConfigurationStoreAware) { - ((ConfigurationStoreAware) metaStoreManagerFactory).setConfigurationStore(configurationStore); - } - RealmEntityManagerFactory entityManagerFactory = - new RealmEntityManagerFactory(metaStoreManagerFactory); - CallContextResolver callContextResolver = configuration.getCallContextResolver(); - callContextResolver.setMetaStoreManagerFactory(metaStoreManagerFactory); - if (callContextResolver instanceof ConfigurationStoreAware csa) { - csa.setConfigurationStore(configurationStore); - } + MetaStoreManagerFactory metaStoreManagerFactory = + configuration.findService(MetaStoreManagerFactory.class); - RealmContextResolver realmContextResolver = configuration.getRealmContextResolver(); - realmContextResolver.setMetaStoreManagerFactory(metaStoreManagerFactory); + // Use the PolarisApplicationConfig to register dependencies in the Jersey resource + // configuration. This uses a different ServiceLocator from the one in the bootstrap step + environment + .getApplicationContext() + .setAttribute(ServletProperties.SERVICE_LOCATOR, configuration.getServiceLocator()); + + environment + .jersey() + .register( + new AbstractBinder() { + @Override + protected void configure() { + bindFactory(CallContextFactory.class).to(CallContext.class).in(RequestScoped.class); + bindFactory(RealmContextFactory.class) + .to(RealmContext.class) + .in(RequestScoped.class); + bind(RealmScopeContext.class) + .in(Singleton.class) + .to(new TypeLiteral>() {}); + bindFactory(PolarisMetaStoreManagerFactory.class) + .to(PolarisMetaStoreManager.class) + .in(RealmScoped.class); + bindFactory(EntityCacheFactory.class).in(RealmScoped.class).to(EntityCache.class); + + bindFactory(PolarisRemoteCacheFactory.class) + .in(RealmScoped.class) + .to(PolarisRemoteCache.class); + + // factory to use a cache delegating grant cache + // currently depends explicitly on the metaStoreManager as the delegate grant + // manager + bindFactory(PolarisMetaStoreManagerFactory.class) + .in(RealmScoped.class) + .to(PolarisGrantManager.class); + polarisMetricRegistry.init( + IcebergRestCatalogApi.class, + IcebergRestConfigurationApi.class, + IcebergRestOAuth2Api.class, + PolarisCatalogsApi.class, + PolarisPrincipalsApi.class, + PolarisPrincipalRolesApi.class); + bindAsContract(RealmEntityManagerFactory.class).in(Singleton.class); + bind(PolarisCallContextCatalogFactory.class) + .to(CallContextCatalogFactory.class) + .in(Singleton.class); + bind(PolarisAuthorizerImpl.class).in(Singleton.class).to(PolarisAuthorizer.class); + bind(IcebergCatalogAdapter.class) + .in(Singleton.class) + .to(IcebergRestCatalogApiService.class) + .to(IcebergRestConfigurationApiService.class); + bind(PolarisServiceImpl.class) + .in(Singleton.class) + .to(PolarisCatalogsApiService.class) + .to(PolarisPrincipalsApiService.class) + .to(PolarisPrincipalRolesApiService.class); + FileIOFactory fileIOFactory = configuration.findService(FileIOFactory.class); + + TaskHandlerConfiguration taskConfig = configuration.getTaskHandler(); + TaskExecutorImpl taskExecutor = + new TaskExecutorImpl(taskConfig.executorService(), metaStoreManagerFactory); + TaskFileIOSupplier fileIOSupplier = + new TaskFileIOSupplier(metaStoreManagerFactory, fileIOFactory); + taskExecutor.addTaskHandler( + new TableCleanupTaskHandler( + taskExecutor, metaStoreManagerFactory, fileIOSupplier)); + taskExecutor.addTaskHandler( + new ManifestFileCleanupTaskHandler( + fileIOSupplier, Executors.newVirtualThreadPerTaskExecutor())); + + bind(taskExecutor).to(TaskExecutor.class); + } + }); + + // servlet filters don't use the underlying DI environment .servlets() .addFilter( - "realmContext", new ContextResolverFilter(realmContextResolver, callContextResolver)) + "realmContext", + new ContextResolverFilter( + configuration.findService(RealmContextResolver.class), + configuration.findService(CallContextResolver.class))) .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); - FileIOFactory fileIOFactory = configuration.getFileIOFactory(); - if (fileIOFactory instanceof MetricRegistryAware mrAware) { - mrAware.setMetricRegistry(polarisMetricRegistry); - } - if (fileIOFactory instanceof OpenTelemetryAware otAware) { - otAware.setOpenTelemetry(openTelemetry); - } - if (fileIOFactory instanceof ConfigurationStoreAware csAware) { - csAware.setConfigurationStore(configurationStore); - } - - TaskHandlerConfiguration taskConfig = configuration.getTaskHandler(); - TaskExecutorImpl taskExecutor = - new TaskExecutorImpl(taskConfig.executorService(), metaStoreManagerFactory); - TaskFileIOSupplier fileIOSupplier = - new TaskFileIOSupplier(metaStoreManagerFactory, fileIOFactory); - taskExecutor.addTaskHandler( - new TableCleanupTaskHandler(taskExecutor, metaStoreManagerFactory, fileIOSupplier)); - taskExecutor.addTaskHandler( - new ManifestFileCleanupTaskHandler( - fileIOSupplier, Executors.newVirtualThreadPerTaskExecutor())); - LOGGER.info( "Initializing PolarisCallContextCatalogFactory for metaStoreManagerType {}", metaStoreManagerFactory); - CallContextCatalogFactory catalogFactory = - new PolarisCallContextCatalogFactory( - entityManagerFactory, metaStoreManagerFactory, taskExecutor, fileIOFactory); - PolarisAuthorizer authorizer = new PolarisAuthorizerImpl(configurationStore); - IcebergCatalogAdapter catalogAdapter = - new IcebergCatalogAdapter( - catalogFactory, entityManagerFactory, metaStoreManagerFactory, authorizer); - environment.jersey().register(new IcebergRestCatalogApi(catalogAdapter)); - environment.jersey().register(new IcebergRestConfigurationApi(catalogAdapter)); + environment.jersey().register(IcebergRestCatalogApi.class); + environment.jersey().register(IcebergRestConfigurationApi.class); FilterRegistration.Dynamic corsRegistration = environment.servlets().addFilter("CORS", CrossOriginFilter.class); @@ -272,18 +382,18 @@ public void run(PolarisApplicationConfig configuration, Environment environment) CrossOriginFilter.ALLOW_CREDENTIALS_PARAM, configuration.getCorsConfiguration().getAllowCredentials()); corsRegistration.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); + + OpenTelemetry openTelemetry = configuration.findService(OpenTelemetry.class); environment .servlets() .addFilter("tracing", new TracingFilter(openTelemetry)) .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); - if (configuration.getRateLimiter() != null) { - environment.jersey().register(new RateLimiterFilter(configuration.getRateLimiter())); + if (configuration.hasRateLimiter()) { + environment.jersey().register(RateLimiterFilter.class); } - - DiscoverableAuthenticator authenticator = - configuration.getPolarisAuthenticator(); - authenticator.setMetaStoreManagerFactory(metaStoreManagerFactory); + Authenticator authenticator = + configuration.findService(new TypeLiteral<>() {}); AuthFilter oauthCredentialAuthFilter = new OAuthCredentialAuthFilter.Builder() .setAuthenticator(authenticator) @@ -291,18 +401,15 @@ public void run(PolarisApplicationConfig configuration, Environment environment) .buildAuthFilter(); environment.jersey().register(new AuthDynamicFeature(oauthCredentialAuthFilter)); environment.healthChecks().register("polaris", new PolarisHealthCheck()); - OAuth2ApiService oauth2Service = configuration.getOauth2Service(); - if (oauth2Service instanceof HasMetaStoreManagerFactory emfAware) { - emfAware.setMetaStoreManagerFactory(metaStoreManagerFactory); - } - environment.jersey().register(new IcebergRestOAuth2Api(oauth2Service)); - environment.jersey().register(new IcebergExceptionMapper()); - environment.jersey().register(new PolarisExceptionMapper()); - PolarisServiceImpl polarisService = - new PolarisServiceImpl(entityManagerFactory, metaStoreManagerFactory, authorizer); - environment.jersey().register(new PolarisCatalogsApi(polarisService)); - environment.jersey().register(new PolarisPrincipalsApi(polarisService)); - environment.jersey().register(new PolarisPrincipalRolesApi(polarisService)); + + environment.jersey().register(IcebergRestOAuth2Api.class); + environment.jersey().register(IcebergExceptionMapper.class); + environment.jersey().register(PolarisExceptionMapper.class); + + environment.jersey().register(PolarisCatalogsApi.class); + environment.jersey().register(PolarisPrincipalsApi.class); + environment.jersey().register(PolarisPrincipalRolesApi.class); + ObjectMapper objectMapper = environment.getObjectMapper(); objectMapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); @@ -317,20 +424,14 @@ public void run(PolarisApplicationConfig configuration, Environment environment) LOGGER.info("Limiting request body size to {} bytes", maxRequestBodyBytes); } - environment.jersey().register(new StreamReadConstraintsExceptionMapper()); + environment.jersey().register(StreamReadConstraintsExceptionMapper.class); RESTSerializers.registerAll(objectMapper); Serializers.registerSerializers(objectMapper); - environment.jersey().register(new IcebergJsonProcessingExceptionMapper()); - environment.jersey().register(new IcebergJerseyViolationExceptionMapper()); - environment.jersey().register(new TimedApplicationEventListener(polarisMetricRegistry)); + environment.jersey().register(IcebergJsonProcessingExceptionMapper.class); + environment.jersey().register(IcebergJerseyViolationExceptionMapper.class); - polarisMetricRegistry.init( - IcebergRestCatalogApi.class, - IcebergRestConfigurationApi.class, - IcebergRestOAuth2Api.class, - PolarisCatalogsApi.class, - PolarisPrincipalsApi.class, - PolarisPrincipalRolesApi.class); + // for tests, we have to instantiate the TimedApplicationEventListener directly + environment.jersey().register(new TimedApplicationEventListener(polarisMetricRegistry)); environment .admin() @@ -378,6 +479,7 @@ private static class ContextResolverFilter implements Filter { private final RealmContextResolver realmContextResolver; private final CallContextResolver callContextResolver; + @Inject public ContextResolverFilter( RealmContextResolver realmContextResolver, CallContextResolver callContextResolver) { this.realmContextResolver = realmContextResolver; @@ -420,4 +522,59 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha } } } + + /** Factory to create a CallContext based on the request contents. */ + @RequestScoped + private static class CallContextFactory implements Factory { + + @RequestScoped + @Override + public CallContext provide() { + return CallContext.getCurrentContext(); + } + + @Override + public void dispose(CallContext instance) {} + } + + @RequestScoped + private static class RealmContextFactory implements Factory { + @Inject Provider callContext; + + @RequestScoped + @Override + public RealmContext provide() { + return callContext.get().getRealmContext(); + } + + @Override + public void dispose(RealmContext instance) {} + } + + private static class PolarisMetaStoreManagerFactory implements Factory { + @Inject MetaStoreManagerFactory metaStoreManagerFactory; + + @RealmScoped + @Override + public PolarisMetaStoreManager provide() { + RealmContext realmContext = CallContext.getCurrentContext().getRealmContext(); + return metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + } + + @Override + public void dispose(PolarisMetaStoreManager instance) {} + } + + private static class PolarisRemoteCacheFactory implements Factory { + @Inject PolarisMetaStoreManager metaStoreManager; + + @RealmScoped + @Override + public PolarisRemoteCache provide() { + return metaStoreManager; + } + + @Override + public void dispose(PolarisRemoteCache instance) {} + } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/PurgeRealmsCommand.java b/polaris-service/src/main/java/org/apache/polaris/service/PurgeRealmsCommand.java index 238a82cbb..5cc05c6ac 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/PurgeRealmsCommand.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/PurgeRealmsCommand.java @@ -21,11 +21,8 @@ import io.dropwizard.core.cli.ConfiguredCommand; import io.dropwizard.core.setup.Bootstrap; import net.sourceforge.argparse4j.inf.Namespace; -import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.ConfigurationStoreAware; import org.apache.polaris.service.config.PolarisApplicationConfig; -import org.apache.polaris.service.context.CallContextResolver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,17 +40,8 @@ protected void run( Namespace namespace, PolarisApplicationConfig configuration) throws Exception { - MetaStoreManagerFactory metaStoreManagerFactory = configuration.getMetaStoreManagerFactory(); - - PolarisConfigurationStore configurationStore = configuration.getConfigurationStore(); - if (metaStoreManagerFactory instanceof ConfigurationStoreAware) { - ((ConfigurationStoreAware) metaStoreManagerFactory).setConfigurationStore(configurationStore); - } - CallContextResolver callContextResolver = configuration.getCallContextResolver(); - callContextResolver.setMetaStoreManagerFactory(metaStoreManagerFactory); - if (callContextResolver instanceof ConfigurationStoreAware csa) { - csa.setConfigurationStore(configurationStore); - } + MetaStoreManagerFactory metaStoreManagerFactory = + configuration.findService(MetaStoreManagerFactory.class); metaStoreManagerFactory.purgeRealms(configuration.getDefaultRealms()); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/TimedApplicationEventListener.java b/polaris-service/src/main/java/org/apache/polaris/service/TimedApplicationEventListener.java index db319d3a8..52d7dbaa7 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/TimedApplicationEventListener.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/TimedApplicationEventListener.java @@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import io.micrometer.core.instrument.Tag; +import jakarta.inject.Inject; import java.lang.reflect.Method; import java.util.List; import java.util.concurrent.TimeUnit; @@ -54,6 +55,7 @@ public class TimedApplicationEventListener implements ApplicationEventListener { // The PolarisMetricRegistry instance used for recording metrics and error counters. private final PolarisMetricRegistry polarisMetricRegistry; + @Inject public TimedApplicationEventListener(PolarisMetricRegistry polarisMetricRegistry) { this.polarisMetricRegistry = polarisMetricRegistry; } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index 148d4944c..7d6901d47 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.admin; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import java.util.List; @@ -82,6 +83,7 @@ public class PolarisServiceImpl private final PolarisAuthorizer polarisAuthorizer; private final MetaStoreManagerFactory metaStoreManagerFactory; + @Inject public PolarisServiceImpl( RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java index c06f17d86..313ff4c55 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.service.auth; +import io.dropwizard.auth.Authenticator; +import jakarta.inject.Inject; import java.util.Arrays; import java.util.HashSet; import java.util.Optional; @@ -46,17 +48,12 @@ * these roles will be active in the current request. */ public abstract class BasePolarisAuthenticator - implements DiscoverableAuthenticator { + implements Authenticator { public static final String PRINCIPAL_ROLE_ALL = "PRINCIPAL_ROLE:ALL"; public static final String PRINCIPAL_ROLE_PREFIX = "PRINCIPAL_ROLE:"; private static final Logger LOGGER = LoggerFactory.getLogger(BasePolarisAuthenticator.class); - protected MetaStoreManagerFactory metaStoreManagerFactory; - - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } + @Inject protected MetaStoreManagerFactory metaStoreManagerFactory; public PolarisCallContext getCurrentPolarisContext() { return CallContext.getCurrentContext().getPolarisCallContext(); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java index 57a9bc74d..2b32aba05 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java @@ -20,7 +20,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.smallrye.common.annotation.Identifier; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import org.apache.commons.codec.binary.Base64; @@ -28,8 +29,6 @@ import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.iceberg.rest.responses.OAuthTokenResponse; import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; import org.apache.polaris.service.config.OAuth2ApiService; import org.apache.polaris.service.types.TokenType; import org.slf4j.Logger; @@ -39,10 +38,11 @@ * Default implementation of the {@link OAuth2ApiService} that generates a JWT token for the client * if the client secret matches. */ -@JsonTypeName("default") -public class DefaultOAuth2ApiService implements OAuth2ApiService, HasMetaStoreManagerFactory { +@Identifier("default") +public class DefaultOAuth2ApiService implements OAuth2ApiService { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultOAuth2ApiService.class); - private TokenBrokerFactory tokenBrokerFactory; + + @Inject private TokenBrokerFactory tokenBrokerFactory; public DefaultOAuth2ApiService() {} @@ -120,16 +120,4 @@ public Response getToken( .build()) .build(); } - - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - if (tokenBrokerFactory instanceof HasMetaStoreManagerFactory hemf) { - hemf.setMetaStoreManagerFactory(metaStoreManagerFactory); - } - } - - @Override - public void setTokenBroker(TokenBrokerFactory tokenBrokerFactory) { - this.tokenBrokerFactory = tokenBrokerFactory; - } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java index b57572d3b..eff07c2a5 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java @@ -18,15 +18,15 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonProperty; +import io.smallrye.common.annotation.Identifier; +import jakarta.inject.Inject; import java.util.Optional; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; +@Identifier("default") public class DefaultPolarisAuthenticator extends BasePolarisAuthenticator { - private TokenBrokerFactory tokenBrokerFactory; + @Inject private TokenBrokerFactory tokenBrokerFactory; @Override public Optional authenticate(String credentials) { @@ -35,18 +35,4 @@ public Optional authenticate(String credentials) DecodedToken decodedToken = handler.verify(credentials); return getPrincipal(decodedToken); } - - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - super.setMetaStoreManagerFactory(metaStoreManagerFactory); - if (tokenBrokerFactory instanceof HasMetaStoreManagerFactory) { - ((HasMetaStoreManagerFactory) tokenBrokerFactory) - .setMetaStoreManagerFactory(metaStoreManagerFactory); - } - } - - @JsonProperty("tokenBroker") - public void setTokenBroker(TokenBrokerFactory tokenBrokerFactory) { - this.tokenBrokerFactory = tokenBrokerFactory; - } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/DiscoverableAuthenticator.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/DiscoverableAuthenticator.java index a6c2129b4..ff76e7de3 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/DiscoverableAuthenticator.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/DiscoverableAuthenticator.java @@ -18,11 +18,9 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.dropwizard.auth.Authenticator; import io.dropwizard.jackson.Discoverable; import java.security.Principal; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; /** * Extension of the {@link Authenticator} interface that extends {@link Discoverable} so @@ -33,6 +31,4 @@ * @param * @param

*/ -@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "class") -public interface DiscoverableAuthenticator - extends Authenticator, Discoverable, HasMetaStoreManagerFactory {} +public interface DiscoverableAuthenticator extends Authenticator {} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java index 5e3441f47..4b11a4666 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java @@ -18,15 +18,16 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.smallrye.common.annotation.Identifier; +import jakarta.inject.Inject; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; -@JsonTypeName("rsa-key-pair") -public class JWTRSAKeyPairFactory implements TokenBrokerFactory, HasMetaStoreManagerFactory { +@Identifier("rsa-key-pair") +public class JWTRSAKeyPairFactory implements TokenBrokerFactory { private int maxTokenGenerationInSeconds = 3600; - private MetaStoreManagerFactory metaStoreManagerFactory; + + @Inject private MetaStoreManagerFactory metaStoreManagerFactory; public void setMaxTokenGenerationInSeconds(int maxTokenGenerationInSeconds) { this.maxTokenGenerationInSeconds = maxTokenGenerationInSeconds; @@ -38,9 +39,4 @@ public TokenBroker apply(RealmContext realmContext) { metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext), maxTokenGenerationInSeconds); } - - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java index 297103f8f..bb04b0482 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java @@ -18,18 +18,18 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.smallrye.common.annotation.Identifier; +import jakarta.inject.Inject; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.function.Supplier; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; -@JsonTypeName("symmetric-key") -public class JWTSymmetricKeyFactory implements TokenBrokerFactory, HasMetaStoreManagerFactory { - private MetaStoreManagerFactory metaStoreManagerFactory; +@Identifier("symmetric-key") +public class JWTSymmetricKeyFactory implements TokenBrokerFactory { + @Inject private MetaStoreManagerFactory metaStoreManagerFactory; private int maxTokenGenerationInSeconds = 3600; private String file; private String secret; @@ -68,7 +68,6 @@ public void setSecret(String secret) { this.secret = secret; } - @Override public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { this.metaStoreManagerFactory = metaStoreManagerFactory; } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/KeyProvider.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/KeyProvider.java index bdd07e9bf..a28b9a059 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/KeyProvider.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/KeyProvider.java @@ -18,13 +18,10 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; import java.security.PrivateKey; import java.security.PublicKey; -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") -public interface KeyProvider extends Discoverable { +public interface KeyProvider { PublicKey getPublicKey(); PrivateKey getPrivateKey(); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java index b37a077d0..d43fffb4c 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java @@ -20,6 +20,7 @@ import com.google.common.base.Splitter; import io.dropwizard.auth.AuthenticationException; +import io.smallrye.common.annotation.Identifier; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -45,6 +46,7 @@ * This class does not expect a client to be either present or correct. Lookup is delegated to the * {@link PolarisMetaStoreManager} for the current realm. */ +@Identifier("test") public class TestInlineBearerTokenPolarisAuthenticator extends BasePolarisAuthenticator { private static final Logger LOGGER = LoggerFactory.getLogger(TestInlineBearerTokenPolarisAuthenticator.class); diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java index a000489f1..ffcdc6a98 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java @@ -18,7 +18,8 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.smallrye.common.annotation.Identifier; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import java.util.HashMap; @@ -32,17 +33,16 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; import org.apache.polaris.service.config.OAuth2ApiService; import org.apache.polaris.service.types.TokenType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@JsonTypeName("test") -public class TestOAuth2ApiService implements OAuth2ApiService, HasMetaStoreManagerFactory { +@Identifier("test") +public class TestOAuth2ApiService implements OAuth2ApiService { private static final Logger LOGGER = LoggerFactory.getLogger(TestOAuth2ApiService.class); - private MetaStoreManagerFactory metaStoreManagerFactory; + @Inject private MetaStoreManagerFactory metaStoreManagerFactory; @Override public Response getToken( @@ -107,12 +107,4 @@ private String getPrincipalName(String clientId) { return principalResult.getEntity().getName(); } } - - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } - - @Override - public void setTokenBroker(TokenBrokerFactory tokenBrokerFactory) {} } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/auth/TokenBrokerFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/auth/TokenBrokerFactory.java index abb15c9db..131f3ed64 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/auth/TokenBrokerFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/auth/TokenBrokerFactory.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.service.auth; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; import java.util.function.Function; import org.apache.polaris.core.context.RealmContext; @@ -27,5 +25,4 @@ * Factory that creates a {@link TokenBroker} for generating and parsing. The {@link TokenBroker} is * created based on the realm context. */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") -public interface TokenBrokerFactory extends Function, Discoverable {} +public interface TokenBrokerFactory extends Function {} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 52e3c5adf..926fcfe81 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import java.net.URLEncoder; @@ -113,6 +114,7 @@ public class IcebergCatalogAdapter private final RealmEntityManagerFactory entityManagerFactory; private final PolarisAuthorizer polarisAuthorizer; + @Inject public IcebergCatalogAdapter( CallContextCatalogFactory catalogFactory, RealmEntityManagerFactory entityManagerFactory, diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java index cf1ee6f87..b930fb35a 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java @@ -18,14 +18,14 @@ */ package org.apache.polaris.service.catalog.io; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.smallrye.common.annotation.Identifier; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.io.FileIO; /** A simple FileIOFactory implementation that defers all the work to the Iceberg SDK */ -@JsonTypeName("default") +@Identifier("default") public class DefaultFileIOFactory implements FileIOFactory { @Override public FileIO loadFileIO(String impl, Map properties) { diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java index 206aaeaa0..64999bbf5 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java @@ -19,7 +19,6 @@ package org.apache.polaris.service.catalog.io; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; import java.util.Map; import org.apache.iceberg.io.FileIO; @@ -28,6 +27,6 @@ use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "factoryType") -public interface FileIOFactory extends Discoverable { +public interface FileIOFactory { FileIO loadFileIO(String impl, Map properties); } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java index 0483a5de4..016afa54c 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java @@ -18,14 +18,14 @@ */ package org.apache.polaris.service.catalog.io; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.smallrye.common.annotation.Identifier; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.io.FileIO; /** A {@link FileIOFactory} that translates WASB paths to ABFS ones */ -@JsonTypeName("wasb") +@Identifier("wasb") public class WasbTranslatingFileIOFactory implements FileIOFactory { @Override public FileIO loadFileIO(String ioImpl, Map properties) { diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/OAuth2ApiService.java b/polaris-service/src/main/java/org/apache/polaris/service/config/OAuth2ApiService.java index 81c219afa..0ab006d8c 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/OAuth2ApiService.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/OAuth2ApiService.java @@ -19,11 +19,7 @@ package org.apache.polaris.service.config; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; -import org.apache.polaris.service.auth.TokenBrokerFactory; import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -public interface OAuth2ApiService extends Discoverable, IcebergRestOAuth2ApiService { - void setTokenBroker(TokenBrokerFactory tokenBrokerFactory); -} +public interface OAuth2ApiService extends IcebergRestOAuth2ApiService {} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java b/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java index 7d5fd8fdf..5f05961f0 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/PolarisApplicationConfig.java @@ -19,14 +19,20 @@ package org.apache.polaris.service.config; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.auth.oauth2.AccessToken; import com.google.auth.oauth2.GoogleCredentials; import com.google.common.base.Preconditions; +import io.dropwizard.auth.Authenticator; import io.dropwizard.core.Configuration; +import jakarta.annotation.Nonnull; +import jakarta.annotation.PostConstruct; +import jakarta.inject.Inject; import java.io.IOException; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.function.Supplier; import javax.annotation.Nullable; @@ -34,26 +40,45 @@ import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.auth.DiscoverableAuthenticator; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +import org.apache.polaris.service.auth.DecodedToken; +import org.apache.polaris.service.auth.TokenBroker; +import org.apache.polaris.service.auth.TokenBrokerFactory; +import org.apache.polaris.service.auth.TokenResponse; +import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.context.CallContextResolver; import org.apache.polaris.service.context.RealmContextResolver; import org.apache.polaris.service.ratelimiter.RateLimiter; +import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; +import org.apache.polaris.service.types.TokenType; +import org.glassfish.hk2.api.Factory; +import org.glassfish.hk2.api.ServiceLocator; +import org.glassfish.hk2.api.TypeLiteral; +import org.glassfish.hk2.utilities.ServiceLocatorUtilities; +import org.glassfish.hk2.utilities.binding.AbstractBinder; import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; /** * Configuration specific to a Polaris REST Service. Place these entries in a YML file for them to * be picked up, i.e. `iceberg-rest-server.yml` */ public class PolarisApplicationConfig extends Configuration { + /** + * Override the default binding of registered services so that the configured instances are used. + */ + private static final int OVERRIDE_BINDING_RANK = 10; + private MetaStoreManagerFactory metaStoreManagerFactory; private String defaultRealm = "default-realm"; private RealmContextResolver realmContextResolver; private CallContextResolver callContextResolver; - private DiscoverableAuthenticator polarisAuthenticator; + private Authenticator polarisAuthenticator; private CorsConfiguration corsConfiguration = new CorsConfiguration(); private TaskHandlerConfiguration taskHandler = new TaskHandlerConfiguration(); private Map globalFeatureConfiguration = Map.of(); @@ -63,57 +88,197 @@ public class PolarisApplicationConfig extends Configuration { private String awsSecretKey; private FileIOFactory fileIOFactory; private RateLimiter rateLimiter; + private TokenBrokerFactory tokenBrokerFactory; private AccessToken gcpAccessToken; public static final long REQUEST_BODY_BYTES_NO_LIMIT = -1; private long maxRequestBodyBytes = REQUEST_BODY_BYTES_NO_LIMIT; + @Inject ServiceLocator serviceLocator; + + @PostConstruct + public void bindToServiceLocator() { + ServiceLocatorUtilities.bind(serviceLocator, binder()); + } + + public ServiceLocator getServiceLocator() { + return serviceLocator; + } + + @Nonnull + public AbstractBinder binder() { + PolarisApplicationConfig config = this; + return new AbstractBinder() { + @Override + protected void configure() { + bindFactory(SupplierFactory.create(serviceLocator, config::getStorageIntegrationProvider)) + .to(PolarisStorageIntegrationProvider.class) + .ranked(OVERRIDE_BINDING_RANK); + bindFactory(SupplierFactory.create(serviceLocator, config::getMetaStoreManagerFactory)) + .to(MetaStoreManagerFactory.class) + .ranked(OVERRIDE_BINDING_RANK); + bindFactory(SupplierFactory.create(serviceLocator, config::createConfigurationStore)) + .to(PolarisConfigurationStore.class) + .ranked(OVERRIDE_BINDING_RANK); + bindFactory(SupplierFactory.create(serviceLocator, config::getFileIOFactory)) + .to(FileIOFactory.class) + .ranked(OVERRIDE_BINDING_RANK); + bindFactory(SupplierFactory.create(serviceLocator, config::getPolarisAuthenticator)) + .to(Authenticator.class) + .ranked(OVERRIDE_BINDING_RANK); + bindFactory(SupplierFactory.create(serviceLocator, config::getTokenBrokerFactory)) + .to(TokenBrokerFactory.class) + .ranked(OVERRIDE_BINDING_RANK); + bindFactory(SupplierFactory.create(serviceLocator, config::getOauth2Service)) + .to(IcebergRestOAuth2ApiService.class) + .ranked(OVERRIDE_BINDING_RANK); + bindFactory(SupplierFactory.create(serviceLocator, config::getCallContextResolver)) + .to(CallContextResolver.class) + .ranked(OVERRIDE_BINDING_RANK); + bindFactory(SupplierFactory.create(serviceLocator, config::getRealmContextResolver)) + .to(RealmContextResolver.class) + .ranked(OVERRIDE_BINDING_RANK); + bindFactory(SupplierFactory.create(serviceLocator, config::getRateLimiter)) + .to(RateLimiter.class) + .ranked(OVERRIDE_BINDING_RANK); + } + }; + } + + /** + * Factory implementation that uses the provided supplier method to retrieve the instance and then + * uses the {@link #serviceLocator} to inject dependencies into the instance. This is necessary + * since the DI framework doesn't automatically inject dependencies into the instances created. + * + * @param + */ + private static final class SupplierFactory implements Factory { + private final ServiceLocator serviceLocator; + private final Supplier supplier; + + private static SupplierFactory create( + ServiceLocator serviceLocator, Supplier supplier) { + return new SupplierFactory<>(serviceLocator, supplier); + } + + private SupplierFactory(ServiceLocator serviceLocator, Supplier supplier) { + this.serviceLocator = serviceLocator; + this.supplier = supplier; + } + + @Override + public T provide() { + T obj = supplier.get(); + serviceLocator.inject(obj); + return obj; + } + + @Override + public void dispose(T instance) {} + } + + public T findService(Class serviceClass) { + return serviceLocator.getService(serviceClass); + } + + public T findService(TypeLiteral serviceClass) { + return serviceLocator.getService(serviceClass.getRawType()); + } + @JsonProperty("metaStoreManager") + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { this.metaStoreManagerFactory = metaStoreManagerFactory; } - @JsonProperty("metaStoreManager") - public MetaStoreManagerFactory getMetaStoreManagerFactory() { + private MetaStoreManagerFactory getMetaStoreManagerFactory() { return metaStoreManagerFactory; } @JsonProperty("io") + @JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "factoryType") public void setFileIOFactory(FileIOFactory fileIOFactory) { this.fileIOFactory = fileIOFactory; } - @JsonProperty("io") - public FileIOFactory getFileIOFactory() { + private FileIOFactory getFileIOFactory() { return fileIOFactory; } @JsonProperty("authenticator") + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "class") public void setPolarisAuthenticator( - DiscoverableAuthenticator polarisAuthenticator) { + Authenticator polarisAuthenticator) { this.polarisAuthenticator = polarisAuthenticator; } - public DiscoverableAuthenticator - getPolarisAuthenticator() { + private Authenticator getPolarisAuthenticator() { return polarisAuthenticator; } - public RealmContextResolver getRealmContextResolver() { + @JsonProperty("tokenBroker") + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") + public void setTokenBrokerFactory(TokenBrokerFactory tokenBrokerFactory) { + this.tokenBrokerFactory = tokenBrokerFactory; + } + + private TokenBrokerFactory getTokenBrokerFactory() { + // return a no-op implementation if none is specified + return Objects.requireNonNullElseGet( + tokenBrokerFactory, + () -> + (rc) -> + new TokenBroker() { + @Override + public boolean supportsGrantType(String grantType) { + return false; + } + + @Override + public boolean supportsRequestedTokenType(TokenType tokenType) { + return false; + } + + @Override + public TokenResponse generateFromClientSecrets( + String clientId, String clientSecret, String grantType, String scope) { + return null; + } + + @Override + public TokenResponse generateFromToken( + TokenType tokenType, String subjectToken, String grantType, String scope) { + return null; + } + + @Override + public DecodedToken verify(String token) { + return null; + } + }); + } + + private RealmContextResolver getRealmContextResolver() { realmContextResolver.setDefaultRealm(this.defaultRealm); return realmContextResolver; } + @JsonProperty("realmContextResolver") + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") public void setRealmContextResolver(RealmContextResolver realmContextResolver) { this.realmContextResolver = realmContextResolver; } - public CallContextResolver getCallContextResolver() { + private CallContextResolver getCallContextResolver() { return callContextResolver; } @JsonProperty("callContextResolver") + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") public void setCallContextResolver(CallContextResolver callContextResolver) { this.callContextResolver = callContextResolver; } @@ -121,11 +286,12 @@ public void setCallContextResolver(CallContextResolver callContextResolver) { private OAuth2ApiService oauth2Service; @JsonProperty("oauth2") + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") public void setOauth2Service(OAuth2ApiService oauth2Service) { this.oauth2Service = oauth2Service; } - public OAuth2ApiService getOauth2Service() { + private OAuth2ApiService getOauth2Service() { return oauth2Service; } @@ -150,11 +316,16 @@ public void setCorsConfiguration(CorsConfiguration corsConfiguration) { } @JsonProperty("rateLimiter") - public RateLimiter getRateLimiter() { + private RateLimiter getRateLimiter() { return rateLimiter; } + public boolean hasRateLimiter() { + return rateLimiter != null; + } + @JsonProperty("rateLimiter") + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") public void setRateLimiter(@Nullable RateLimiter rateLimiter) { this.rateLimiter = rateLimiter; } @@ -193,7 +364,7 @@ public long getMaxRequestBodyBytes() { return maxRequestBodyBytes; } - public PolarisConfigurationStore getConfigurationStore() { + private PolarisConfigurationStore createConfigurationStore() { return new DefaultConfigurationStore(globalFeatureConfiguration, realmConfiguration); } @@ -201,7 +372,7 @@ public List getDefaultRealms() { return defaultRealms; } - public AwsCredentialsProvider credentialsProvider() { + private AwsCredentialsProvider credentialsProvider() { if (StringUtils.isNotBlank(awsAccessKey) && StringUtils.isNotBlank(awsSecretKey)) { LoggerFactory.getLogger(PolarisApplicationConfig.class) .warn("Using hard-coded AWS credentials - this is not recommended for production"); @@ -223,7 +394,31 @@ public void setDefaultRealms(List defaultRealms) { this.defaultRealms = defaultRealms; } - public Supplier getGcpCredentialsProvider() { + private PolarisStorageIntegrationProvider storageIntegrationProvider; + + public void setStorageIntegrationProvider( + PolarisStorageIntegrationProvider storageIntegrationProvider) { + this.storageIntegrationProvider = storageIntegrationProvider; + } + + private PolarisStorageIntegrationProvider getStorageIntegrationProvider() { + if (storageIntegrationProvider == null) { + storageIntegrationProvider = + new PolarisStorageIntegrationProviderImpl( + () -> { + StsClientBuilder stsClientBuilder = StsClient.builder(); + AwsCredentialsProvider awsCredentialsProvider = credentialsProvider(); + if (awsCredentialsProvider != null) { + stsClientBuilder.credentialsProvider(awsCredentialsProvider); + } + return stsClientBuilder.build(); + }, + getGcpCredentialsProvider()); + } + return storageIntegrationProvider; + } + + private Supplier getGcpCredentialsProvider() { return () -> Optional.ofNullable(gcpAccessToken) .map(GoogleCredentials::create) diff --git a/polaris-service/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java index b896a13eb..fc4e0ca47 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java @@ -18,11 +18,14 @@ */ package org.apache.polaris.service.config; +import jakarta.inject.Inject; +import jakarta.inject.Provider; +import java.util.HashMap; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; +import org.apache.polaris.core.persistence.cache.EntityCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,15 +35,20 @@ public class RealmEntityManagerFactory { private final MetaStoreManagerFactory metaStoreManagerFactory; // Key: realmIdentifier - private final Map cachedEntityManagers = new ConcurrentHashMap<>(); + private final Map cachedEntityManagers = new HashMap<>(); + private final Provider entityCache; // Subclasses for test injection. protected RealmEntityManagerFactory() { this.metaStoreManagerFactory = null; + this.entityCache = null; } - public RealmEntityManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { + @Inject + public RealmEntityManagerFactory( + MetaStoreManagerFactory metaStoreManagerFactory, Provider entityCache) { this.metaStoreManagerFactory = metaStoreManagerFactory; + this.entityCache = entityCache; } public PolarisEntityManager getOrCreateEntityManager(RealmContext context) { @@ -54,7 +62,8 @@ public PolarisEntityManager getOrCreateEntityManager(RealmContext context) { LOGGER.info("Initializing new PolarisEntityManager for realm {}", r); return new PolarisEntityManager( metaStoreManagerFactory.getOrCreateMetaStoreManager(context), - metaStoreManagerFactory.getOrCreateStorageCredentialCache(context)); + metaStoreManagerFactory.getOrCreateStorageCredentialCache(context), + entityCache.get()); }); } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/context/CallContextResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/context/CallContextResolver.java index 6221e73f0..71b21d0f3 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/context/CallContextResolver.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/context/CallContextResolver.java @@ -18,16 +18,12 @@ */ package org.apache.polaris.service.context; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; import java.util.Map; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; /** Uses the resolved RealmContext to further resolve elements of the CallContext. */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -public interface CallContextResolver extends HasMetaStoreManagerFactory, Discoverable { +public interface CallContextResolver { CallContext resolveCallContext( RealmContext realmContext, String method, diff --git a/polaris-service/src/main/java/org/apache/polaris/service/context/DefaultContextResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/context/DefaultContextResolver.java index 520d8fdcd..540384180 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/context/DefaultContextResolver.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/context/DefaultContextResolver.java @@ -18,8 +18,9 @@ */ package org.apache.polaris.service.context; -import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Splitter; +import io.smallrye.common.annotation.Identifier; +import jakarta.inject.Inject; import java.time.Clock; import java.time.ZoneId; import java.util.HashMap; @@ -32,7 +33,6 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; -import org.apache.polaris.service.config.ConfigurationStoreAware; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,9 +42,8 @@ * *

Example: principal:data-engineer;password:test;realm:acct123 */ -@JsonTypeName("default") -public class DefaultContextResolver - implements RealmContextResolver, CallContextResolver, ConfigurationStoreAware { +@Identifier("default") +public class DefaultContextResolver implements RealmContextResolver, CallContextResolver { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultContextResolver.class); public static final String REALM_PROPERTY_KEY = "realm"; @@ -52,20 +51,10 @@ public class DefaultContextResolver public static final String PRINCIPAL_PROPERTY_KEY = "principal"; public static final String PRINCIPAL_PROPERTY_DEFAULT_VALUE = "default-principal"; - private MetaStoreManagerFactory metaStoreManagerFactory; - private PolarisConfigurationStore configurationStore; + @Inject private MetaStoreManagerFactory metaStoreManagerFactory; + @Inject private PolarisConfigurationStore configurationStore; private String defaultRealm = "default-realm"; - /** - * During CallContext resolution that might depend on RealmContext, the {@code - * entityManagerFactory} will be used to resolve elements of the CallContext which require - * additional information from an underlying entity store. - */ - @Override - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } - @Override public RealmContext resolveRealmContext( String requestURL, @@ -165,9 +154,4 @@ private static Map parseBearerTokenAsKvPairs(Map } return parsedProperties; } - - @Override - public void setConfigurationStore(PolarisConfigurationStore configurationStore) { - this.configurationStore = configurationStore; - } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java index 83d659a83..a72f71431 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.context; +import jakarta.inject.Inject; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; @@ -50,6 +51,7 @@ public class PolarisCallContextCatalogFactory implements CallContextCatalogFacto private final FileIOFactory fileIOFactory; private final MetaStoreManagerFactory metaStoreManagerFactory; + @Inject public PolarisCallContextCatalogFactory( RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, diff --git a/polaris-service/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java b/polaris-service/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java index 681a10b60..66e0c99da 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java @@ -18,14 +18,10 @@ */ package org.apache.polaris.service.context; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; import java.util.Map; import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.service.config.HasMetaStoreManagerFactory; -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -public interface RealmContextResolver extends Discoverable, HasMetaStoreManagerFactory { +public interface RealmContextResolver { RealmContext resolveRealmContext( String requestURL, diff --git a/polaris-service/src/main/java/org/apache/polaris/service/context/RealmScopeContext.java b/polaris-service/src/main/java/org/apache/polaris/service/context/RealmScopeContext.java new file mode 100644 index 000000000..f5b33fa3f --- /dev/null +++ b/polaris-service/src/main/java/org/apache/polaris/service/context/RealmScopeContext.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.context; + +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import java.lang.annotation.Annotation; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.context.RealmScoped; +import org.glassfish.hk2.api.ActiveDescriptor; +import org.glassfish.hk2.api.Context; +import org.glassfish.hk2.api.IterableProvider; +import org.glassfish.hk2.api.ServiceHandle; +import org.glassfish.hk2.api.ServiceLocator; + +@Singleton +public class RealmScopeContext implements Context { + private final Map, Object>> contexts = new ConcurrentHashMap<>(); + + @Inject private ServiceLocator locator; + @Inject private IterableProvider realmContextProvider; + + @Override + public Class getScope() { + return RealmScoped.class; + } + + @SuppressWarnings("unchecked") + @Override + public U findOrCreate(ActiveDescriptor activeDescriptor, ServiceHandle root) { + RealmContext realmContext = realmContextProvider.iterator().next(); + Map, Object> contextMap = + contexts.computeIfAbsent(realmContext.getRealmIdentifier(), k -> new ConcurrentHashMap<>()); + return (U) contextMap.computeIfAbsent(activeDescriptor, k -> activeDescriptor.create(root)); + } + + @Override + public boolean containsKey(ActiveDescriptor descriptor) { + RealmContext realmContext = realmContextProvider.iterator().next(); + Map, Object> contextMap = + contexts.computeIfAbsent(realmContext.getRealmIdentifier(), k -> new HashMap<>()); + return contextMap.containsKey(descriptor); + } + + @Override + public void destroyOne(ActiveDescriptor descriptor) { + RealmContext realmContext = realmContextProvider.iterator().next(); + Map, Object> contextMap = + contexts.computeIfAbsent(realmContext.getRealmIdentifier(), k -> new HashMap<>()); + contextMap.remove(descriptor); + } + + @Override + public boolean supportsNullCreation() { + return false; + } + + @Override + public boolean isActive() { + Optional first = + locator.getAllServices(Context.class).stream() + .filter( + context -> + context + .getScope() + .equals( + realmContextProvider + .getHandle() + .getActiveDescriptor() + .getScopeAnnotation())) + .findFirst(); + return first.map(Context::isActive).orElse(false); + } + + @Override + public void shutdown() { + contexts.clear(); + } +} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java index fc830341e..7714fc3df 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java @@ -18,9 +18,10 @@ */ package org.apache.polaris.service.persistence; -import com.fasterxml.jackson.annotation.JsonTypeName; -import io.dropwizard.jackson.Discoverable; +import com.google.common.annotations.VisibleForTesting; +import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nonnull; +import jakarta.inject.Inject; import java.util.Collections; import java.util.HashSet; import java.util.Map; @@ -34,10 +35,13 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl; import org.apache.polaris.core.persistence.PolarisTreeMapStore; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; -@JsonTypeName("in-memory") +@Identifier("in-memory") public class InMemoryPolarisMetaStoreManagerFactory - extends LocalPolarisMetaStoreManagerFactory implements Discoverable { + extends LocalPolarisMetaStoreManagerFactory { + @Inject protected PolarisStorageIntegrationProvider storageIntegration; + final Set bootstrappedRealms = new HashSet<>(); @Override @@ -87,4 +91,10 @@ private void bootstrapRealmAndPrintCredentials(String realmId) { principalSecrets.getPrincipalSecrets().getMainSecret()); System.out.println(msg); } + + @VisibleForTesting + public void setStorageIntegrationProvider( + PolarisStorageIntegrationProvider storageIntegrationProvider) { + this.storageIntegration = storageIntegrationProvider; + } } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/persistence/cache/EntityCacheFactory.java b/polaris-service/src/main/java/org/apache/polaris/service/persistence/cache/EntityCacheFactory.java new file mode 100644 index 000000000..13be186c7 --- /dev/null +++ b/polaris-service/src/main/java/org/apache/polaris/service/persistence/cache/EntityCacheFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.persistence.cache; + +import jakarta.inject.Inject; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmScoped; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.cache.EntityCache; +import org.glassfish.hk2.api.Factory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EntityCacheFactory implements Factory { + private static Logger LOGGER = LoggerFactory.getLogger(EntityCacheFactory.class); + @Inject PolarisMetaStoreManager metaStoreManager; + + @RealmScoped + @Override + public EntityCache provide() { + LOGGER.debug( + "Creating new EntityCache instance for realm {}", + CallContext.getCurrentContext().getRealmContext().getRealmIdentifier()); + return new EntityCache(metaStoreManager); + } + + @Override + public void dispose(EntityCache instance) { + // no-op + } +} diff --git a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java index 33dffd7b2..7323c46b3 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java @@ -18,10 +18,10 @@ */ package org.apache.polaris.service.ratelimiter; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.smallrye.common.annotation.Identifier; /** Rate limiter that always allows the request */ -@JsonTypeName("no-op") +@Identifier("no-op") public class NoOpRateLimiter implements RateLimiter { @Override public boolean tryAcquire() { diff --git a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiter.java b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiter.java index 5d102b628..be2017d32 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiter.java @@ -18,12 +18,8 @@ */ package org.apache.polaris.service.ratelimiter; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.dropwizard.jackson.Discoverable; - /** Interface for rate limiting requests */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -public interface RateLimiter extends Discoverable { +public interface RateLimiter { /** * This signifies that a request is being made. That is, the rate limiter should count the request * at this point. diff --git a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java index 034717c4c..c5ad957bf 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.ratelimiter; +import jakarta.inject.Inject; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.core.Response; @@ -33,6 +34,7 @@ public class RateLimiterFilter implements ContainerRequestFilter { private final RateLimiter rateLimiter; + @Inject public RateLimiterFilter(RateLimiter rateLimiter) { this.rateLimiter = rateLimiter; } diff --git a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java index f3973fef1..85ea54e22 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java @@ -20,8 +20,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.annotations.VisibleForTesting; +import io.smallrye.common.annotation.Identifier; import java.time.Clock; import java.util.Map; import java.util.Optional; @@ -33,7 +33,7 @@ * Rate limiter that maps the request's realm identifier to its own TokenBucketRateLimiter, with its * own capacity. */ -@JsonTypeName("realm-token-bucket") +@Identifier("realm-token-bucket") public class RealmTokenBucketRateLimiter implements RateLimiter { private final long requestsPerSecond; private final long windowSeconds; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiter.java b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiter.java index 39a8aea87..2b3adb618 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/ratelimiter/TokenBucketRateLimiter.java @@ -18,12 +18,14 @@ */ package org.apache.polaris.service.ratelimiter; +import io.smallrye.common.annotation.Identifier; import java.time.InstantSource; /** * Token bucket implementation of a Polaris RateLimiter. Acquires tokens at a fixed rate and has a * maximum amount of tokens. Each successful "tryAcquire" costs 1 token. */ +@Identifier("token-bucket") public class TokenBucketRateLimiter implements RateLimiter { private final double tokensPerMilli; private final long maxTokens; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java index fad2e42d5..98195c0e5 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java @@ -22,6 +22,7 @@ import com.google.auth.http.HttpTransportFactory; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.ServiceOptions; +import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import java.util.EnumMap; @@ -39,6 +40,7 @@ import org.apache.polaris.core.storage.gcp.GcpCredentialsStorageIntegration; import software.amazon.awssdk.services.sts.StsClient; +@Identifier("default") public class PolarisStorageIntegrationProviderImpl implements PolarisStorageIntegrationProvider { private final Supplier stsClientSupplier; diff --git a/polaris-service/src/main/java/org/apache/polaris/service/tracing/TracingFilter.java b/polaris-service/src/main/java/org/apache/polaris/service/tracing/TracingFilter.java index b3cefe1cb..f5d968296 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/tracing/TracingFilter.java +++ b/polaris-service/src/main/java/org/apache/polaris/service/tracing/TracingFilter.java @@ -28,6 +28,7 @@ import io.opentelemetry.semconv.ServerAttributes; import io.opentelemetry.semconv.UrlAttributes; import jakarta.annotation.Priority; +import jakarta.inject.Inject; import jakarta.servlet.Filter; import jakarta.servlet.FilterChain; import jakarta.servlet.ServletException; @@ -53,6 +54,7 @@ public class TracingFilter implements Filter { private static final Logger LOGGER = LoggerFactory.getLogger(TracingFilter.class); private final OpenTelemetry openTelemetry; + @Inject public TracingFilter(OpenTelemetry openTelemetry) { this.openTelemetry = openTelemetry; } diff --git a/polaris-service/src/main/resources/META-INF/hk2-locator/default b/polaris-service/src/main/resources/META-INF/hk2-locator/default new file mode 100644 index 000000000..58dec4c27 --- /dev/null +++ b/polaris-service/src/main/resources/META-INF/hk2-locator/default @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +[org.apache.polaris.service.auth.DefaultPolarisAuthenticator]S +contract={org.apache.polaris.service.auth.BasePolarisAuthenticator} +name=default +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator]S +contract={io.dropwizard.auth.Authenticator} +name=test +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory]S +contract={org.apache.polaris.core.persistence.MetaStoreManagerFactory} +name=in-memory +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.service.auth.DefaultOAuth2ApiService]S +contract={org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService} +name=default +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.service.auth.TestOAuth2ApiService]S +contract={org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService} +name=test +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.service.auth.JWTSymmetricKeyFactory]S +contract={org.apache.polaris.service.auth.TokenBrokerFactory} +name=symmetric-key +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.service.auth.JWTRSAKeyPairFactory]S +contract={org.apache.polaris.service.auth.TokenBrokerFactory} +name=rsa-key-pair +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.service.context.DefaultContextResolver]S +contract={org.apache.polaris.service.context.CallContextResolver,org.apache.polaris.service.context.RealmContextResolver} +name=default +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.core.storage.PolarisStorageIntegrationProvider]S +contract={org.apache.polaris.core.storage.PolarisStorageIntegrationProvider} +name=default +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.service.catalog.io.DefaultFileIOFactory]S +contract={org.apache.polaris.service.catalog.io.FileIOFactory} +name=default +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.service.catalog.io.WasbTranslatingFileIOFactory]S +contract={org.apache.polaris.service.catalog.io.FileIOFactory} +name=wasb +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.service.ratelimiter.NoOpRateLimiter]S +contract={org.apache.polaris.service.ratelimiter.RateLimiter} +name=no-op +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.service.ratelimiter.TokenBucketRateLimiter]S +contract={org.apache.polaris.service.ratelimiter.RateLimiter} +name=token-bucket +qualifier={io.smallrye.common.annotation.Identifier} + +[org.apache.polaris.service.ratelimiter.RealmTokenBucketRateLimiter]S +contract={org.apache.polaris.service.ratelimiter.RateLimiter} +name=realm-token-bucket +qualifier={io.smallrye.common.annotation.Identifier} + diff --git a/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable b/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable deleted file mode 100644 index 4c7ab2acc..000000000 --- a/polaris-service/src/main/resources/META-INF/services/io.dropwizard.jackson.Discoverable +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.auth.DiscoverableAuthenticator -org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory -org.apache.polaris.service.config.OAuth2ApiService -org.apache.polaris.service.context.RealmContextResolver -org.apache.polaris.service.context.CallContextResolver -org.apache.polaris.service.auth.TokenBrokerFactory -org.apache.polaris.service.catalog.io.FileIOFactory -org.apache.polaris.service.ratelimiter.RateLimiter diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.auth.TokenBrokerFactory b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.auth.TokenBrokerFactory deleted file mode 100644 index 422b154c7..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.auth.TokenBrokerFactory +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.auth.JWTRSAKeyPairFactory -org.apache.polaris.service.auth.JWTSymmetricKeyFactory \ No newline at end of file diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory deleted file mode 100644 index 6b280ad71..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.catalog.io.DefaultFileIOFactory -org.apache.polaris.service.catalog.io.WasbTranslatingFileIOFactory diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.config.OAuth2ApiService b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.config.OAuth2ApiService deleted file mode 100644 index 3c8f0e254..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.config.OAuth2ApiService +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.auth.TestOAuth2ApiService -org.apache.polaris.service.auth.DefaultOAuth2ApiService \ No newline at end of file diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.CallContextResolver b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.CallContextResolver deleted file mode 100644 index 1ac9dbea2..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.CallContextResolver +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.context.DefaultContextResolver \ No newline at end of file diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.RealmContextResolver b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.RealmContextResolver deleted file mode 100644 index 1ac9dbea2..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.context.RealmContextResolver +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.context.DefaultContextResolver \ No newline at end of file diff --git a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter b/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter deleted file mode 100644 index 461bcc2db..000000000 --- a/polaris-service/src/main/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.ratelimiter.RealmTokenBucketRateLimiter -org.apache.polaris.service.ratelimiter.NoOpRateLimiter diff --git a/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationConfigurationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationConfigurationTest.java index 74442e040..f92f1ef00 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationConfigurationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/PolarisApplicationConfigurationTest.java @@ -24,6 +24,7 @@ import io.dropwizard.testing.ResourceHelpers; import io.dropwizard.testing.junit5.DropwizardAppExtension; import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory; import org.apache.polaris.service.config.PolarisApplicationConfig; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; @@ -50,7 +51,7 @@ class DefaultMetastore { @Test void testMetastoreType() { - assertThat(app.getConfiguration().getMetaStoreManagerFactory()) + assertThat(app.getConfiguration().findService(MetaStoreManagerFactory.class)) .isInstanceOf(InMemoryPolarisMetaStoreManagerFactory.class); } } @@ -69,7 +70,7 @@ class EclipseLinkMetastore { @Test void testMetastoreType() { - assertThat(app.getConfiguration().getMetaStoreManagerFactory()) + assertThat(app.getConfiguration().findService(MetaStoreManagerFactory.class)) .isInstanceOf(EclipseLinkPolarisMetaStoreManagerFactory.class) .extracting("persistenceUnitName", "confFile") .containsExactly("test-unit", "/test-conf-file"); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java index ce28d4b54..ecc9a01fb 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java @@ -64,6 +64,7 @@ import org.apache.polaris.core.entity.PrincipalRoleEntity; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.catalog.BasePolarisCatalog; @@ -173,7 +174,9 @@ public void before() { } }, Clock.systemDefaultZone()); - this.entityManager = new PolarisEntityManager(metaStoreManager, new StorageCredentialCache()); + this.entityManager = + new PolarisEntityManager( + metaStoreManager, new StorageCredentialCache(), new EntityCache(metaStoreManager)); this.metaStoreManager = metaStoreManager; callContext = CallContext.of(realmContext, polarisContext); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTest.java index bb60f7149..7e040a4da 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisOverlappingTableTest.java @@ -36,6 +36,7 @@ import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.requests.CreateTableRequest; import org.apache.polaris.core.PolarisConfiguration; +import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogProperties; import org.apache.polaris.core.admin.model.CreateCatalogRequest; @@ -102,7 +103,7 @@ public String catalog() { private String extensionName() { return (extension .getConfiguration() - .getConfigurationStore() + .findService(PolarisConfigurationStore.class) .getConfiguration(null, PolarisConfiguration.ALLOW_TABLE_LOCATION_OVERLAP)) ? "lax" : "strict"; diff --git a/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisRealmEntityCacheTest.java b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisRealmEntityCacheTest.java new file mode 100644 index 000000000..f35747ce3 --- /dev/null +++ b/polaris-service/src/test/java/org/apache/polaris/service/admin/PolarisRealmEntityCacheTest.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.admin; + +import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY; +import static org.assertj.core.api.Assertions.assertThat; + +import io.dropwizard.core.setup.Environment; +import io.dropwizard.testing.ConfigOverride; +import io.dropwizard.testing.ResourceHelpers; +import io.dropwizard.testing.junit5.DropwizardAppExtension; +import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import jakarta.inject.Inject; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.Invocation; +import jakarta.ws.rs.core.Feature; +import jakarta.ws.rs.core.FeatureContext; +import jakarta.ws.rs.core.Response; +import java.io.IOException; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.CreateCatalogRequest; +import org.apache.polaris.core.admin.model.CreateCatalogRoleRequest; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.PolarisBaseEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.cache.EntityCacheByNameKey; +import org.apache.polaris.core.persistence.cache.EntityCacheEntry; +import org.apache.polaris.service.PolarisApplication; +import org.apache.polaris.service.config.PolarisApplicationConfig; +import org.apache.polaris.service.test.PolarisConnectionExtension; +import org.apache.polaris.service.test.PolarisRealm; +import org.apache.polaris.service.test.TestEnvironmentExtension; +import org.glassfish.hk2.api.ServiceLocator; +import org.glassfish.jersey.process.internal.RequestScope; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * tests around the {@link org.apache.polaris.service.persistence.cache.EntityCacheFactory} and + * ensuring that the {@link org.apache.polaris.core.persistence.cache.EntityCache} is managed per + * realm. + */ +@ExtendWith({ + DropwizardExtensionsSupport.class, + TestEnvironmentExtension.class, + PolarisConnectionExtension.class +}) +public class PolarisRealmEntityCacheTest { + private static ServiceLocatorAccessor accessor = new ServiceLocatorAccessor(); + + /** + * Injectable {@link Feature} that allows us to access the {@link ServiceLocator} from the jersey + * resource configuration. + */ + private static final class ServiceLocatorAccessor implements Feature { + @Inject ServiceLocator serviceLocator; + + @Override + public boolean configure(FeatureContext context) { + return true; + } + + public ServiceLocator getServiceLocator() { + return serviceLocator; + } + } + + private static final DropwizardAppExtension EXT = + new DropwizardAppExtension<>( + PolarisApplication.class, + ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), + ConfigOverride.config( + "server.applicationConnectors[0].port", + "0"), // Bind to random port to support parallelism + ConfigOverride.config("server.adminConnectors[0].port", "0")) + .addListener( + new DropwizardAppExtension.ServiceListener() { + @Override + public void onRun( + PolarisApplicationConfig configuration, + Environment environment, + DropwizardAppExtension rule) + throws Exception { + environment.jersey().register(accessor); + } + }); + private static String userToken; + private static String realm; + + @BeforeAll + public static void setup( + PolarisConnectionExtension.PolarisToken adminToken, @PolarisRealm String polarisRealm) + throws IOException { + userToken = adminToken.token(); + realm = polarisRealm; + + // Set up test location + PolarisConnectionExtension.createTestDir(realm); + } + + @Test + public void testRealmEntityCacheEquality() { + ServiceLocator serviceLocator = accessor.getServiceLocator(); + RequestScope requestScope = serviceLocator.getService(RequestScope.class); + EntityCache cache1; + // check that multiple calls to the serviceLocator return the same instance of the EntityCache + // within the same call context + try (CallContext ctx = + CallContext.setCurrentContext( + CallContext.of(() -> realm, new PolarisCallContext(null, null)))) { + cache1 = requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); + EntityCache cache2 = + requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); + assertThat(cache1).isSameAs(cache2); + } + + // in a new call context with a different realm, the EntityCache should be different + try (CallContext ctx = + CallContext.setCurrentContext( + CallContext.of(() -> "anotherrealm", new PolarisCallContext(null, null)))) { + EntityCache cache2 = + requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); + assertThat(cache1).isNotSameAs(cache2); + } + + // but if we start a new call context with the original realm, we'll get the same EntityCache + // instance + try (CallContext ctx = + CallContext.setCurrentContext( + CallContext.of(() -> realm, new PolarisCallContext(null, null)))) { + EntityCache cache2 = + requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); + assertThat(cache1).isSameAs(cache2); + } + } + + @Test + public void testCacheForRealm() { + // create a catalog + String catalogName = "mycachecatalog"; + ServiceLocator serviceLocator = accessor.getServiceLocator(); + RequestScope requestScope = serviceLocator.getService(RequestScope.class); + listCatalogs(); + + // check for the catalog - it should not exist + // the service_admin role, however, should exist + try (CallContext ctx = + CallContext.setCurrentContext( + CallContext.of(() -> realm, new PolarisCallContext(null, null)))) { + EntityCache cache = + requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); + assertThat(cache).isNotNull(); + EntityCacheEntry cachedCatalog = + cache.getEntityByName(new EntityCacheByNameKey(PolarisEntityType.CATALOG, catalogName)); + assertThat(cachedCatalog).isNull(); + EntityCacheEntry serviceAdmin = + cache.getEntityByName( + new EntityCacheByNameKey( + PolarisEntityType.PRINCIPAL_ROLE, + PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())); + assertThat(serviceAdmin) + .isNotNull() + .extracting(EntityCacheEntry::getEntity) + .returns(PolarisEntityType.PRINCIPAL_ROLE, PolarisBaseEntity::getType) + .returns( + PolarisEntityConstants.getNameOfPrincipalServiceAdminRole(), + PolarisBaseEntity::getName); + } + Catalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setStorageConfigInfo( + new AwsStorageConfigInfo( + "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) + .setProperties(new CatalogProperties("s3://bucket1/")) + .build(); + createCatalog(catalog); + createCatalogRole(catalogName, "my_cr", userToken); + + // now check again for the catalog - it should exist + try (CallContext ctx = + CallContext.setCurrentContext( + CallContext.of(() -> realm, new PolarisCallContext(null, null)))) { + EntityCache cache = + requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); + assertThat(cache).isNotNull(); + EntityCacheEntry cachedCatalog = + cache.getEntityByName(new EntityCacheByNameKey(PolarisEntityType.CATALOG, catalogName)); + assertThat(cachedCatalog) + .isNotNull() + .extracting(EntityCacheEntry::getEntity) + .returns(catalogName, PolarisBaseEntity::getName); + } + + // but if we check a different realm, the catalog should not exist in the cache + // the service_admin role also does not exist, since it's never been used + try (CallContext ctx = + CallContext.setCurrentContext( + CallContext.of(() -> "another-realm", new PolarisCallContext(null, null)))) { + EntityCache cache = + requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); + assertThat(cache).isNotNull(); + EntityCacheEntry cachedCatalog = + cache.getEntityByName(new EntityCacheByNameKey(PolarisEntityType.CATALOG, catalogName)); + assertThat(cachedCatalog).isNull(); + EntityCacheEntry serviceAdmin = + cache.getEntityByName( + new EntityCacheByNameKey( + PolarisEntityType.PRINCIPAL_ROLE, + PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())); + assertThat(serviceAdmin).isNull(); + } + } + + private static Invocation.Builder newRequest(String url, String token) { + return EXT.client() + .target(String.format(url, EXT.getLocalPort())) + .request("application/json") + .header("Authorization", "Bearer " + token) + .header(REALM_PROPERTY_KEY, realm); + } + + private static Invocation.Builder newRequest(String url) { + return newRequest(url, userToken); + } + + private static void listCatalogs() { + try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + } + } + + private static void createCatalog(Catalog catalog) { + try (Response response = + newRequest("http://localhost:%d/api/management/v1/catalogs") + .post(Entity.json(new CreateCatalogRequest(catalog)))) { + + assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + } + + private static void createCatalogRole( + String catalogName, String catalogRoleName, String catalogAdminToken) { + try (Response response = + newRequest( + "http://localhost:%d/api/management/v1/catalogs/" + catalogName + "/catalog-roles", + catalogAdminToken) + .post(Entity.json(new CreateCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { + assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + } +} diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java index 704c7a4ef..53583dc2f 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogTest.java @@ -76,11 +76,11 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.core.monitor.PolarisMetricRegistry; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.storage.PolarisCredentialProperty; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; @@ -157,7 +157,9 @@ public void before() { } }, Clock.systemDefaultZone()); - entityManager = new PolarisEntityManager(metaStoreManager, new StorageCredentialCache()); + entityManager = + new PolarisEntityManager( + metaStoreManager, new StorageCredentialCache(), new EntityCache(metaStoreManager)); CallContext callContext = CallContext.of(realmContext, polarisContext); CallContext.setCurrentContext(callContext); @@ -287,9 +289,6 @@ public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext rea return new StorageCredentialCache(); } - @Override - public void setMetricRegistry(PolarisMetricRegistry metricRegistry) {} - @Override public Map bootstrapRealms(List realms) { throw new NotImplementedException("Bootstrapping realms is not supported"); @@ -299,10 +298,6 @@ public Map bootstrapRealms(List realms) public void purgeRealms(List realms) { throw new NotImplementedException("Purging realms is not supported"); } - - @Override - public void setStorageIntegrationProvider( - PolarisStorageIntegrationProvider storageIntegrationProvider) {} }; } diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java index 6385cd94b..3df944fa9 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/BasePolarisCatalogViewTest.java @@ -49,6 +49,7 @@ import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; @@ -89,7 +90,8 @@ public void before() { Clock.systemDefaultZone()); PolarisEntityManager entityManager = - new PolarisEntityManager(metaStoreManager, new StorageCredentialCache()); + new PolarisEntityManager( + metaStoreManager, new StorageCredentialCache(), new EntityCache(metaStoreManager)); CallContext callContext = CallContext.of(null, polarisContext); CallContext.setCurrentContext(callContext); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/FileIOIntegrationTest.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/FileIOIntegrationTest.java index 9b0f2d7fd..30a2d9e2e 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/FileIOIntegrationTest.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/FileIOIntegrationTest.java @@ -95,7 +95,7 @@ public static void beforeAll( PolarisConnectionExtension.PolarisToken adminToken, SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, @PolarisRealm String realm) { - ioFactory = (TestFileIOFactory) EXT.getConfiguration().getFileIOFactory(); + ioFactory = (TestFileIOFactory) EXT.getConfiguration().findService(FileIOFactory.class); FileStorageConfigInfo storageConfigInfo = FileStorageConfigInfo.builder() diff --git a/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/TestFileIOFactory.java b/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/TestFileIOFactory.java index 9afeeb155..4cc8e72f2 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/TestFileIOFactory.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/catalog/io/TestFileIOFactory.java @@ -18,7 +18,7 @@ */ package org.apache.polaris.service.catalog.io; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.smallrye.common.annotation.Identifier; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -32,7 +32,7 @@ * A FileIOFactory that measures the number of bytes read, files written, and files deleted. It can * inject exceptions at various parts of the IO construction. */ -@JsonTypeName("test") +@Identifier("test") public class TestFileIOFactory implements FileIOFactory { private final List ios = new ArrayList<>(); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/MockRealmTokenBucketRateLimiter.java b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/MockRealmTokenBucketRateLimiter.java index 95570091b..a4c4e66bd 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/MockRealmTokenBucketRateLimiter.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/MockRealmTokenBucketRateLimiter.java @@ -20,14 +20,14 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; +import io.smallrye.common.annotation.Identifier; import java.time.Clock; import java.time.Instant; import java.time.ZoneOffset; import org.threeten.extra.MutableClock; /** RealmTokenBucketRateLimiter with a mock clock */ -@JsonTypeName("mock-realm-token-bucket") +@Identifier("mock-realm-token-bucket") public class MockRealmTokenBucketRateLimiter extends RealmTokenBucketRateLimiter { public static MutableClock CLOCK = MutableClock.of(Instant.now(), ZoneOffset.UTC); diff --git a/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java b/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java index cca61c8a8..8d49c0836 100644 --- a/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java +++ b/polaris-service/src/test/java/org/apache/polaris/service/test/PolarisConnectionExtension.java @@ -42,6 +42,8 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.service.auth.TokenUtils; import org.apache.polaris.service.config.PolarisApplicationConfig; +import org.apache.polaris.service.context.CallContextResolver; +import org.apache.polaris.service.context.RealmContextResolver; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; @@ -79,7 +81,7 @@ public void beforeAll(ExtensionContext extensionContext) throws Exception { try { PolarisApplicationConfig config = (PolarisApplicationConfig) dropwizardAppExtension.getConfiguration(); - metaStoreManagerFactory = config.getMetaStoreManagerFactory(); + metaStoreManagerFactory = config.findService(MetaStoreManagerFactory.class); if (!(metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { metaStoreManagerFactory.bootstrapRealms(List.of(realm)); } @@ -92,7 +94,7 @@ public void beforeAll(ExtensionContext extensionContext) throws Exception { RealmContext realmContext = config - .getRealmContextResolver() + .findService(RealmContextResolver.class) .resolveRealmContext( String.format("%s://%s", testEnvUri.getScheme(), testEnvUri.getHost()), "GET", @@ -101,7 +103,7 @@ public void beforeAll(ExtensionContext extensionContext) throws Exception { Map.of(REALM_PROPERTY_KEY, realm)); CallContext ctx = config - .getCallContextResolver() + .findService(CallContextResolver.class) .resolveCallContext(realmContext, "GET", path, Map.of(), Map.of()); CallContext.setCurrentContext(ctx); PolarisMetaStoreManager metaStoreManager = diff --git a/polaris-service/src/main/java/org/apache/polaris/service/tracing/OpenTelemetryAware.java b/polaris-service/src/test/resources/META-INF/hk2-locator/default similarity index 66% rename from polaris-service/src/main/java/org/apache/polaris/service/tracing/OpenTelemetryAware.java rename to polaris-service/src/test/resources/META-INF/hk2-locator/default index 07a24e29c..a456b6a7a 100644 --- a/polaris-service/src/main/java/org/apache/polaris/service/tracing/OpenTelemetryAware.java +++ b/polaris-service/src/test/resources/META-INF/hk2-locator/default @@ -16,11 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.tracing; +[org.apache.polaris.service.catalog.io.TestFileIOFactory]S +contract={org.apache.polaris.service.catalog.io.FileIOFactory} +name=test +qualifier={io.smallrye.common.annotation.Identifier} -import io.opentelemetry.api.OpenTelemetry; - -/** Allows setting a configured instance of {@link OpenTelemetry} */ -public interface OpenTelemetryAware { - void setOpenTelemetry(OpenTelemetry openTelemetry); -} +[org.apache.polaris.service.ratelimiter.MockRealmTokenBucketRateLimiter]S +contract={org.apache.polaris.service.ratelimiter.RateLimiter} +name=mock-realm-token-bucket +qualifier={io.smallrye.common.annotation.Identifier} diff --git a/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.auth.DiscoverableAuthenticator b/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.auth.DiscoverableAuthenticator deleted file mode 100644 index c8652a626..000000000 --- a/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.auth.DiscoverableAuthenticator +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator \ No newline at end of file diff --git a/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory b/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory deleted file mode 100644 index 21db576ff..000000000 --- a/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.catalog.io.FileIOFactory +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.catalog.io.TestFileIOFactory \ No newline at end of file diff --git a/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter b/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter deleted file mode 100644 index c43c88a2a..000000000 --- a/polaris-service/src/test/resources/META-INF/services/org.apache.polaris.service.ratelimiter.RateLimiter +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -org.apache.polaris.service.ratelimiter.MockRealmTokenBucketRateLimiter diff --git a/polaris-service/src/test/resources/polaris-server-integrationtest.yml b/polaris-service/src/test/resources/polaris-server-integrationtest.yml index b3c8289a7..76db1ec1c 100644 --- a/polaris-service/src/test/resources/polaris-server-integrationtest.yml +++ b/polaris-service/src/test/resources/polaris-server-integrationtest.yml @@ -87,16 +87,13 @@ io: oauth2: type: default - tokenBroker: - type: symmetric-key - secret: polaris authenticator: class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator - tokenBroker: - type: symmetric-key - secret: polaris +tokenBroker: + type: symmetric-key + secret: polaris callContextResolver: type: default diff --git a/regtests/Dockerfile b/regtests/Dockerfile index b0598aeaf..f515e802d 100644 --- a/regtests/Dockerfile +++ b/regtests/Dockerfile @@ -54,4 +54,4 @@ USER root RUN chmod -R go+rwx /home/spark/regtests USER spark -CMD ["./run.sh"] +ENTRYPOINT ["./run.sh"] diff --git a/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py b/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py index a6c216a63..9ae9a1a39 100644 --- a/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py +++ b/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py @@ -40,7 +40,6 @@ from polaris.management import PolarisDefaultApi, Principal, PrincipalRole, CatalogRole, \ CatalogGrant, CatalogPrivilege, ApiException, CreateCatalogRoleRequest, CreatePrincipalRoleRequest, \ CreatePrincipalRequest, AddGrantRequest, GrantCatalogRoleRequest, GrantPrincipalRoleRequest, UpdateCatalogRequest -from polaris.management.exceptions import ForbiddenException @pytest.fixture @@ -160,59 +159,60 @@ def snowman_catalog_client(polaris_catalog_url, snowman): return IcebergCatalogAPI(CatalogApiClient(Configuration(access_token=token.access_token, host=polaris_catalog_url))) + @pytest.fixture def creator_catalog_client(polaris_catalog_url, creator): - """ - Create an iceberg catalog client with TABLE_CREATE credentials - :param polaris_catalog_url: - :param creator: - :return: - """ - client = CatalogApiClient(Configuration(username=creator.principal.client_id, - password=creator.credentials.client_secret, - host=polaris_catalog_url)) - oauth_api = IcebergOAuth2API(client) - token = oauth_api.get_token(scope='PRINCIPAL_ROLE:ALL', client_id=creator.principal.client_id, - client_secret=creator.credentials.client_secret, - grant_type='client_credentials', - _headers={'realm': 'default-realm'}) + """ + Create an iceberg catalog client with TABLE_CREATE credentials + :param polaris_catalog_url: + :param creator: + :return: + """ + client = CatalogApiClient(Configuration(username=creator.principal.client_id, + password=creator.credentials.client_secret, + host=polaris_catalog_url)) + oauth_api = IcebergOAuth2API(client) + token = oauth_api.get_token(scope='PRINCIPAL_ROLE:ALL', client_id=creator.principal.client_id, + client_secret=creator.credentials.client_secret, + grant_type='client_credentials', + _headers={'realm': 'default-realm'}) - return IcebergCatalogAPI(CatalogApiClient(Configuration(access_token=token.access_token, - host=polaris_catalog_url))) + return IcebergCatalogAPI(CatalogApiClient(Configuration(access_token=token.access_token, + host=polaris_catalog_url))) @pytest.fixture def creator(polaris_url, polaris_catalog_url, root_client, snowflake_catalog): - """ - create the creator principal with only TABLE_CREATE privileges - :param root_client: - :param snowflake_catalog: - :return: - """ - creator_name = "creator" - principal_role = "creator_principal_role" - catalog_role = "creator_catalog_role" - try: - creator = create_principal(polaris_url, polaris_catalog_url, root_client, creator_name) - creator_principal_role = create_principal_role(root_client, principal_role) - creator_catalog_role = create_catalog_role(root_client, snowflake_catalog, catalog_role) - - root_client.assign_catalog_role_to_principal_role(principal_role_name=creator_principal_role.name, - catalog_name=snowflake_catalog.name, - grant_catalog_role_request=GrantCatalogRoleRequest( - catalog_role=creator_catalog_role)) - root_client.add_grant_to_catalog_role(snowflake_catalog.name, creator_catalog_role.name, - AddGrantRequest(grant=CatalogGrant(catalog_name=snowflake_catalog.name, - type='catalog', - privilege=CatalogPrivilege.TABLE_CREATE))) - root_client.assign_principal_role(creator.principal.name, - grant_principal_role_request=GrantPrincipalRoleRequest( - principal_role=creator_principal_role)) - yield creator - finally: - root_client.delete_principal(creator_name) - root_client.delete_principal_role(principal_role_name=principal_role) - root_client.delete_catalog_role(catalog_role_name=catalog_role, catalog_name=snowflake_catalog.name) + """ + create the creator principal with only TABLE_CREATE privileges + :param root_client: + :param snowflake_catalog: + :return: + """ + creator_name = "creator" + principal_role = "creator_principal_role" + catalog_role = "creator_catalog_role" + try: + creator = create_principal(polaris_url, polaris_catalog_url, root_client, creator_name) + creator_principal_role = create_principal_role(root_client, principal_role) + creator_catalog_role = create_catalog_role(root_client, snowflake_catalog, catalog_role) + + root_client.assign_catalog_role_to_principal_role(principal_role_name=creator_principal_role.name, + catalog_name=snowflake_catalog.name, + grant_catalog_role_request=GrantCatalogRoleRequest( + catalog_role=creator_catalog_role)) + root_client.add_grant_to_catalog_role(snowflake_catalog.name, creator_catalog_role.name, + AddGrantRequest(grant=CatalogGrant(catalog_name=snowflake_catalog.name, + type='catalog', + privilege=CatalogPrivilege.TABLE_CREATE))) + root_client.assign_principal_role(creator.principal.name, + grant_principal_role_request=GrantPrincipalRoleRequest( + principal_role=creator_principal_role)) + yield creator + finally: + root_client.delete_principal(creator_name) + root_client.delete_principal_role(principal_role_name=principal_role) + root_client.delete_catalog_role(catalog_role_name=catalog_role, catalog_name=snowflake_catalog.name) @pytest.fixture @@ -236,7 +236,8 @@ def reader_catalog_client(polaris_catalog_url, reader): host=polaris_catalog_url))) -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') def test_spark_credentials(root_client, snowflake_catalog, polaris_catalog_url, snowman, reader): """ Basic spark test - using snowman, create namespaces and a table. Insert into the table and read records back. @@ -298,8 +299,10 @@ def test_spark_credentials(root_client, snowflake_catalog, polaris_catalog_url, spark.sql('DROP NAMESPACE db1') -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') -def test_spark_cannot_create_table_outside_of_namespace_dir(root_client, snowflake_catalog, polaris_catalog_url, snowman, reader): +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') +def test_spark_cannot_create_table_outside_of_namespace_dir(root_client, snowflake_catalog, polaris_catalog_url, + snowman, reader): """ Basic spark test - using snowman, create a namespace and try to create a table outside of the namespace. This should fail @@ -327,8 +330,10 @@ def test_spark_cannot_create_table_outside_of_namespace_dir(root_client, snowfla assert "is not in the list of allowed locations" in e.java_exception.getMessage() -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') -def test_spark_creates_table_in_custom_namespace_dir(root_client, snowflake_catalog, polaris_catalog_url, snowman, reader): +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') +def test_spark_creates_table_in_custom_namespace_dir(root_client, snowflake_catalog, polaris_catalog_url, snowman, + reader): """ Basic spark test - using snowman, create a namespace and try to create a table outside of the namespace. This should fail @@ -351,12 +356,15 @@ def test_spark_creates_table_in_custom_namespace_dir(root_client, snowflake_cata spark.sql(f"CREATE TABLE table_in_custom_namespace_location (col1 int, col2 string)") assert spark.sql("SELECT * FROM table_in_custom_namespace_location").count() == 0 # check the metadata and assert the custom namespace location is used - entries = spark.sql(f"SELECT file FROM db1.schema.table_in_custom_namespace_location.metadata_log_entries").collect() + entries = spark.sql( + f"SELECT file FROM db1.schema.table_in_custom_namespace_location.metadata_log_entries").collect() assert namespace_location in entries[0][0] -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') -def test_spark_can_create_table_in_custom_allowed_dir(root_client, snowflake_catalog, polaris_catalog_url, snowman, reader): +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') +def test_spark_can_create_table_in_custom_allowed_dir(root_client, snowflake_catalog, polaris_catalog_url, snowman, + reader): """ Basic spark test - using snowman, create a namespace and try to create a table outside of the namespace. This should fail @@ -374,15 +382,18 @@ def test_spark_can_create_table_in_custom_allowed_dir(root_client, snowflake_cat table_location = snowflake_catalog.properties.default_base_location + '/db1/custom_schema_location/table_outside_namespace' spark.sql(f'USE {snowflake_catalog.name}') spark.sql('CREATE NAMESPACE db1') - spark.sql(f"CREATE NAMESPACE db1.schema LOCATION '{snowflake_catalog.properties.default_base_location}/db1/custom_schema_location'") + spark.sql( + f"CREATE NAMESPACE db1.schema LOCATION '{snowflake_catalog.properties.default_base_location}/db1/custom_schema_location'") spark.sql('SHOW NAMESPACES') spark.sql('USE db1.schema') # this is supported because it is inside of the custom namespace location spark.sql(f"CREATE TABLE iceberg_table_outside_namespace (col1 int, col2 string) LOCATION '{table_location}'") -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') -def test_spark_cannot_create_view_overlapping_table(root_client, snowflake_catalog, polaris_catalog_url, snowman, reader): +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') +def test_spark_cannot_create_view_overlapping_table(root_client, snowflake_catalog, polaris_catalog_url, snowman, + reader): """ Basic spark test - using snowman, create a namespace and try to create a table outside of the namespace. This should fail @@ -405,13 +416,15 @@ def test_spark_cannot_create_view_overlapping_table(root_client, snowflake_catal spark.sql('USE db1.schema') spark.sql(f"CREATE TABLE my_iceberg_table (col1 int, col2 string) LOCATION '{table_location}'") try: - spark.sql(f"CREATE VIEW disallowed_view (int, string) TBLPROPERTIES ('location'= '{table_location}') AS SELECT * FROM my_iceberg_table") + spark.sql( + f"CREATE VIEW disallowed_view (int, string) TBLPROPERTIES ('location'= '{table_location}') AS SELECT * FROM my_iceberg_table") pytest.fail("Expected to fail when creating table outside of namespace directory") except Py4JJavaError as e: assert "conflicts with existing table or namespace at location" in e.java_exception.getMessage() -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') def test_spark_credentials_can_delete_after_purge(root_client, snowflake_catalog, polaris_catalog_url, snowman, snowman_catalog_client, test_bucket): """ @@ -513,9 +526,10 @@ def test_spark_credentials_can_delete_after_purge(root_client, snowflake_catalog pytest.fail(f"Expected all data to be deleted, but found data files {objects['Contents']}") -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') def test_spark_credentials_can_write_with_random_prefix(root_client, snowflake_catalog, polaris_catalog_url, snowman, - snowman_catalog_client, test_bucket): + snowman_catalog_client, test_bucket): """ Update the catalog configuration to support unstructured table locations. Using snowman, create namespaces and a table configured to use object-store layout in a folder under the catalog root, outside of the default table @@ -532,8 +546,9 @@ def test_spark_credentials_can_write_with_random_prefix(root_client, snowflake_c """ snowflake_catalog.properties.additional_properties['allow.unstructured.table.location'] = 'true' root_client.update_catalog(catalog_name=snowflake_catalog.name, - update_catalog_request=UpdateCatalogRequest(properties=snowflake_catalog.properties.to_dict(), - current_entity_version=snowflake_catalog.entity_version)) + update_catalog_request=UpdateCatalogRequest( + properties=snowflake_catalog.properties.to_dict(), + current_entity_version=snowflake_catalog.entity_version)) with IcebergSparkSession(credentials=f'{snowman.principal.client_id}:{snowman.credentials.client_secret}', catalog_name=snowflake_catalog.name, polaris_url=polaris_catalog_url) as spark: @@ -543,7 +558,8 @@ def test_spark_credentials_can_write_with_random_prefix(root_client, snowflake_c spark.sql('CREATE NAMESPACE db1.schema') spark.sql('SHOW NAMESPACES') spark.sql('USE db1.schema') - spark.sql(f"CREATE TABLE {table_name} (col1 int, col2 string) TBLPROPERTIES ('write.object-storage.enabled'='true','write.data.path'='s3://{test_bucket}/polaris_test/snowflake_catalog/{table_name}data')") + spark.sql( + f"CREATE TABLE {table_name} (col1 int, col2 string) TBLPROPERTIES ('write.object-storage.enabled'='true','write.data.path'='s3://{test_bucket}/polaris_test/snowflake_catalog/{table_name}data')") spark.sql('SHOW TABLES') # several inserts and an update, which should cause earlier files to show up as deleted in the later manifests @@ -589,7 +605,7 @@ def test_spark_credentials_can_write_with_random_prefix(root_client, snowflake_c objs_to_delete = [] for prefix in objects['CommonPrefixes']: data_objects = s3.list_objects(Bucket=test_bucket, Delimiter='/', - Prefix=f'{prefix["Prefix"]}schema/{table_name}/') + Prefix=f'{prefix["Prefix"]}schema/{table_name}/') assert data_objects is not None print(data_objects) assert 'Contents' in data_objects @@ -612,9 +628,10 @@ def test_spark_credentials_can_write_with_random_prefix(root_client, snowflake_c Delete={'Objects': objs_to_delete}) -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') def test_spark_object_store_layout_under_table_dir(root_client, snowflake_catalog, polaris_catalog_url, snowman, - snowman_catalog_client, test_bucket): + snowman_catalog_client, test_bucket): """ Using snowman, create namespaces and a table configured to use object-store layout, using a folder under the default table directory structure. Insert into the table in multiple operations and update existing records @@ -639,7 +656,8 @@ def test_spark_object_store_layout_under_table_dir(root_client, snowflake_catalo spark.sql('SHOW NAMESPACES') spark.sql('USE db1.schema') table_base_dir = f'polaris_test/snowflake_catalog/db1/schema/{table_name}/obj_layout/' - spark.sql(f"CREATE TABLE {table_name} (col1 int, col2 string) TBLPROPERTIES ('write.object-storage.enabled'='true','write.data.path'='s3://{test_bucket}/{table_base_dir}')") + spark.sql( + f"CREATE TABLE {table_name} (col1 int, col2 string) TBLPROPERTIES ('write.object-storage.enabled'='true','write.data.path'='s3://{test_bucket}/{table_base_dir}')") spark.sql('SHOW TABLES') # several inserts and an update, which should cause earlier files to show up as deleted in the later manifests @@ -685,7 +703,7 @@ def test_spark_object_store_layout_under_table_dir(root_client, snowflake_catalo objs_to_delete = [] for prefix in objects['CommonPrefixes']: data_objects = s3.list_objects(Bucket=test_bucket, Delimiter='/', - Prefix=f'{prefix["Prefix"]}') + Prefix=f'{prefix["Prefix"]}') assert data_objects is not None print(data_objects) assert 'Contents' in data_objects @@ -708,7 +726,8 @@ def test_spark_object_store_layout_under_table_dir(root_client, snowflake_catalo Delete={'Objects': objs_to_delete}) -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') # @pytest.mark.skip(reason="This test is flaky") def test_spark_credentials_can_create_views(snowflake_catalog, polaris_catalog_url, snowman): """ @@ -765,7 +784,8 @@ def test_spark_credentials_can_create_views(snowflake_catalog, polaris_catalog_u assert view_records[5][0] == 'changed string' -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') def test_spark_credentials_s3_direct_with_write(root_client, snowflake_catalog, polaris_catalog_url, snowman, snowman_catalog_client, test_bucket): """ @@ -792,9 +812,9 @@ def test_spark_credentials_s3_direct_with_write(root_client, snowflake_catalog, table2_metadata = snowman_catalog_client.load_table(snowflake_catalog.name, unquote('db1%1Fschema'), "iceberg_table_2", - "s3_direct_with_write_table2").metadata_location + "vended-credentials").metadata_location response = snowman_catalog_client.load_table(snowflake_catalog.name, unquote('db1%1Fschema'), "iceberg_table", - "s3_direct_with_write") + "vended-credentials") assert response.config is not None assert 's3.access-key-id' in response.config assert 's3.secret-access-key' in response.config @@ -827,8 +847,8 @@ def test_spark_credentials_s3_direct_with_write(root_client, snowflake_catalog, put_object = s3.put_object(Bucket=test_bucket, Key=f"{metadata_file['Key']}.bak", Body=metadata_contents['Body'].read()) assert put_object is not None - assert 'VersionId' in put_object - assert put_object['VersionId'] is not None + assert 'ETag' in put_object + assert put_object['ETag'] is not None # list files in the other table's directory. The access policy should restrict this try: @@ -856,7 +876,8 @@ def test_spark_credentials_s3_direct_with_write(root_client, snowflake_catalog, spark.sql('DROP NAMESPACE db1') -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'false').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'false').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') def test_spark_credentials_s3_direct_without_write(root_client, snowflake_catalog, polaris_catalog_url, snowman, reader_catalog_client, test_bucket): """ @@ -884,10 +905,10 @@ def test_spark_credentials_s3_direct_without_write(root_client, snowflake_catalo table2_metadata = reader_catalog_client.load_table(snowflake_catalog.name, unquote('db1%1Fschema'), "iceberg_table_2", - "s3_direct_with_write_table2").metadata_location + "vended-credentials").metadata_location response = reader_catalog_client.load_table(snowflake_catalog.name, unquote('db1%1Fschema'), "iceberg_table", - "s3_direct_without_write") + "vended-credentials") assert response.config is not None assert 's3.access-key-id' in response.config assert 's3.secret-access-key' in response.config @@ -945,31 +966,32 @@ def test_spark_credentials_s3_direct_without_write(root_client, snowflake_catalo spark.sql('DROP NAMESPACE db1') -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'false').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'false').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') def test_spark_credentials_s3_direct_without_read( - snowflake_catalog, snowman_catalog_client, creator_catalog_client, test_bucket): + snowflake_catalog, snowman_catalog_client, creator_catalog_client, test_bucket): """ Create a table using `creator`, which does not have TABLE_READ_DATA and expect a `ForbiddenException` """ snowman_catalog_client.create_namespace( - prefix=snowflake_catalog.name, - create_namespace_request=CreateNamespaceRequest( - namespace=["some_schema"] - ) + prefix=snowflake_catalog.name, + create_namespace_request=CreateNamespaceRequest( + namespace=["some_schema"] + ) ) try: creator_catalog_client.create_table( - prefix=snowflake_catalog.name, - namespace="some_schema", - x_iceberg_access_delegation="true", - create_table_request=CreateTableRequest( - name="some_table", - var_schema=ModelSchema( - type = 'struct', - fields = [], - ) + prefix=snowflake_catalog.name, + namespace="some_schema", + x_iceberg_access_delegation="true", + create_table_request=CreateTableRequest( + name="some_table", + var_schema=ModelSchema( + type='struct', + fields=[], ) + ) ) pytest.fail("Expected exception when creating a table without TABLE_WRITE") except Exception as e: @@ -1006,106 +1028,111 @@ def create_principal(polaris_url, polaris_catalog_url, api, principal_name): else: raise e -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') + +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') def test_spark_credentials_s3_scoped_to_metadata_data_locations(root_client, snowflake_catalog, polaris_catalog_url, - snowman, snowman_catalog_client, test_bucket): - """ - Create a table using Spark. Then call the loadTable api directly with snowman token to fetch the vended credentials - for the table. - Verify that the credentials returned to snowman can only work for the location that ending with metadata or data directory - :param root_client: - :param snowflake_catalog: - :param polaris_catalog_url: - :param snowman_catalog_client: - :param reader_catalog_client: - :return: - """ - with IcebergSparkSession(credentials=f'{snowman.principal.client_id}:{snowman.credentials.client_secret}', - catalog_name=snowflake_catalog.name, - polaris_url=polaris_catalog_url) as spark: - spark.sql(f'USE {snowflake_catalog.name}') - spark.sql('CREATE NAMESPACE db1') - spark.sql('CREATE NAMESPACE db1.schema') - spark.sql('USE db1.schema') - spark.sql('CREATE TABLE iceberg_table_scope_loc(col1 int, col2 string)') - spark.sql(f'''CREATE TABLE iceberg_table_scope_loc_slashes (col1 int, col2 string) LOCATION \'s3://{test_bucket}/polaris_test/snowflake_catalog/db1/schema/iceberg_table_scope_loc_slashes/path_with_slashes///////\'''') - - prefix1 = 'polaris_test/snowflake_catalog/db1/schema/iceberg_table_scope_loc' - prefix2 = 'polaris_test/snowflake_catalog/db1/schema/iceberg_table_scope_loc_slashes/path_with_slashes' - response1 = snowman_catalog_client.load_table(snowflake_catalog.name, unquote('db1%1Fschema'), - "iceberg_table_scope_loc", - "s3_scoped_table_locations") - response2 = snowman_catalog_client.load_table(snowflake_catalog.name, unquote('db1%1Fschema'), - "iceberg_table_scope_loc_slashes", - "s3_scoped_table_locations_with_slashes") - assert response1 is not None - assert response2 is not None - assert response1.metadata_location.startswith(f"s3://{test_bucket}/{prefix1}/metadata/") - # ensure that the slashes are removed before "/metadata/" - assert response2.metadata_location.startswith(f"s3://{test_bucket}/{prefix2}/metadata/") - - s3_1 = boto3.client('s3', + snowman, snowman_catalog_client, test_bucket): + """ + Create a table using Spark. Then call the loadTable api directly with snowman token to fetch the vended credentials + for the table. + Verify that the credentials returned to snowman can only work for the location that ending with metadata or data directory + :param root_client: + :param snowflake_catalog: + :param polaris_catalog_url: + :param snowman_catalog_client: + :param reader_catalog_client: + :return: + """ + with IcebergSparkSession(credentials=f'{snowman.principal.client_id}:{snowman.credentials.client_secret}', + catalog_name=snowflake_catalog.name, + polaris_url=polaris_catalog_url) as spark: + spark.sql(f'USE {snowflake_catalog.name}') + spark.sql('CREATE NAMESPACE db1') + spark.sql('CREATE NAMESPACE db1.schema') + spark.sql('USE db1.schema') + spark.sql('CREATE TABLE iceberg_table_scope_loc(col1 int, col2 string)') + spark.sql( + f'''CREATE TABLE iceberg_table_scope_loc_slashes (col1 int, col2 string) LOCATION \'s3://{test_bucket}/polaris_test/snowflake_catalog/db1/schema/iceberg_table_scope_loc_slashes/path_with_slashes///////\'''') + + prefix1 = 'polaris_test/snowflake_catalog/db1/schema/iceberg_table_scope_loc' + prefix2 = 'polaris_test/snowflake_catalog/db1/schema/iceberg_table_scope_loc_slashes/path_with_slashes' + response1 = snowman_catalog_client.load_table(snowflake_catalog.name, unquote('db1%1Fschema'), + "iceberg_table_scope_loc", + "vended-credentials") + response2 = snowman_catalog_client.load_table(snowflake_catalog.name, unquote('db1%1Fschema'), + "iceberg_table_scope_loc_slashes", + "vended-credentials") + assert response1 is not None + assert response2 is not None + assert response1.metadata_location.startswith(f"s3://{test_bucket}/{prefix1}/metadata/") + # ensure that the slashes are removed before "/metadata/" + assert response2.metadata_location.startswith(f"s3://{test_bucket}/{prefix2}/metadata/") + + s3_1 = boto3.client('s3', aws_access_key_id=response1.config['s3.access-key-id'], aws_secret_access_key=response1.config['s3.secret-access-key'], aws_session_token=response1.config['s3.session-token']) - s3_2 = boto3.client('s3', - aws_access_key_id=response2.config['s3.access-key-id'], - aws_secret_access_key=response2.config['s3.secret-access-key'], - aws_session_token=response2.config['s3.session-token']) - for client,prefix in [(s3_1,prefix1), (s3_2, prefix2)]: - objects = client.list_objects(Bucket=test_bucket, Delimiter='/', - Prefix=f'{prefix}/metadata/') - assert objects is not None - assert 'Contents' in objects , f'list metadata files failed in prefix: {prefix}/metadata/' - - objects = client.list_objects(Bucket=test_bucket, Delimiter='/', - Prefix=f'{prefix}/data/') - assert objects is not None - # no insert executed, so should not have any data files - assert 'Contents' not in objects , f'No contents should be in prefix: {prefix}/data/' - - objects = client.list_objects(Bucket=test_bucket, Delimiter='/', + s3_2 = boto3.client('s3', + aws_access_key_id=response2.config['s3.access-key-id'], + aws_secret_access_key=response2.config['s3.secret-access-key'], + aws_session_token=response2.config['s3.session-token']) + for client, prefix in [(s3_1, prefix1), (s3_2, prefix2)]: + objects = client.list_objects(Bucket=test_bucket, Delimiter='/', + Prefix=f'{prefix}/metadata/') + assert objects is not None + assert 'Contents' in objects, f'list metadata files failed in prefix: {prefix}/metadata/' + + objects = client.list_objects(Bucket=test_bucket, Delimiter='/', + Prefix=f'{prefix}/data/') + assert objects is not None + # no insert executed, so should not have any data files + assert 'Contents' not in objects, f'No contents should be in prefix: {prefix}/data/' + + objects = client.list_objects(Bucket=test_bucket, Delimiter='/', Prefix=f'{prefix}/') - assert objects is not None - assert 'CommonPrefixes' in objects , f'list prefixes failed in prefix: {prefix}/' - assert len(objects['CommonPrefixes']) > 0 - - with IcebergSparkSession(credentials=f'{snowman.principal.client_id}:{snowman.credentials.client_secret}', - catalog_name=snowflake_catalog.name, - polaris_url=polaris_catalog_url) as spark: - spark.sql(f'USE {snowflake_catalog.name}') - spark.sql('USE db1.schema') - spark.sql('DROP TABLE iceberg_table_scope_loc PURGE') - spark.sql('DROP TABLE iceberg_table_scope_loc_slashes PURGE') - spark.sql(f'USE {snowflake_catalog.name}') - spark.sql('DROP NAMESPACE db1.schema') - spark.sql('DROP NAMESPACE db1') - -@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', reason='AWS_TEST_ENABLED is not set or is false') + assert objects is not None + assert 'CommonPrefixes' in objects, f'list prefixes failed in prefix: {prefix}/' + assert len(objects['CommonPrefixes']) > 0 + + with IcebergSparkSession(credentials=f'{snowman.principal.client_id}:{snowman.credentials.client_secret}', + catalog_name=snowflake_catalog.name, + polaris_url=polaris_catalog_url) as spark: + spark.sql(f'USE {snowflake_catalog.name}') + spark.sql('USE db1.schema') + spark.sql('DROP TABLE iceberg_table_scope_loc PURGE') + spark.sql('DROP TABLE iceberg_table_scope_loc_slashes PURGE') + spark.sql(f'USE {snowflake_catalog.name}') + spark.sql('DROP NAMESPACE db1.schema') + spark.sql('DROP NAMESPACE db1') + + +@pytest.mark.skipif(os.environ.get('AWS_TEST_ENABLED', 'False').lower() != 'true', + reason='AWS_TEST_ENABLED is not set or is false') def test_spark_ctas(snowflake_catalog, polaris_catalog_url, snowman): - """ - Create a table using CTAS and ensure that credentials are vended - :param root_client: - :param snowflake_catalog: - :return: - """ - with IcebergSparkSession(credentials=f'{snowman.principal.client_id}:{snowman.credentials.client_secret}', - catalog_name=snowflake_catalog.name, - polaris_url=polaris_catalog_url) as spark: - table_name = f'iceberg_test_table_{str(uuid.uuid4())[-10:]}' - spark.sql(f'USE {snowflake_catalog.name}') - spark.sql('CREATE NAMESPACE db1') - spark.sql('CREATE NAMESPACE db1.schema') - spark.sql('USE db1.schema') - spark.sql(f'CREATE TABLE {table_name}_t1 (col1 int)') - spark.sql('SHOW TABLES') - - # Insert some data - spark.sql(f"INSERT INTO {table_name}_t1 VALUES (10)") - - # Run CTAS - spark.sql(f"CREATE TABLE {table_name}_t2 AS SELECT * FROM {table_name}_t1") + """ + Create a table using CTAS and ensure that credentials are vended + :param root_client: + :param snowflake_catalog: + :return: + """ + with IcebergSparkSession(credentials=f'{snowman.principal.client_id}:{snowman.credentials.client_secret}', + catalog_name=snowflake_catalog.name, + polaris_url=polaris_catalog_url) as spark: + table_name = f'iceberg_test_table_{str(uuid.uuid4())[-10:]}' + spark.sql(f'USE {snowflake_catalog.name}') + spark.sql('CREATE NAMESPACE db1') + spark.sql('CREATE NAMESPACE db1.schema') + spark.sql('USE db1.schema') + spark.sql(f'CREATE TABLE {table_name}_t1 (col1 int)') + spark.sql('SHOW TABLES') + + # Insert some data + spark.sql(f"INSERT INTO {table_name}_t1 VALUES (10)") + + # Run CTAS + spark.sql(f"CREATE TABLE {table_name}_t2 AS SELECT * FROM {table_name}_t1") def create_catalog_role(api, catalog, role_name): diff --git a/server-templates/api.mustache b/server-templates/api.mustache index bf16d0f3e..72c16ac8b 100644 --- a/server-templates/api.mustache +++ b/server-templates/api.mustache @@ -52,6 +52,8 @@ import {{javaxPackage}}.servlet.http.HttpServletResponse; import {{javaxPackage}}.ws.rs.core.Context; import {{javaxPackage}}.ws.rs.core.SecurityContext; +import {{javaxPackage}}.inject.Inject; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,6 +85,7 @@ public class {{classname}} { private final {{classname}}Service service; + @Inject public {{classname}}({{classname}}Service service) { this.service = service; }