diff --git a/dlt/destinations/impl/clickhouse/clickhouse.py b/dlt/destinations/impl/clickhouse/clickhouse.py index b534b5f504..307bbaaf46 100644 --- a/dlt/destinations/impl/clickhouse/clickhouse.py +++ b/dlt/destinations/impl/clickhouse/clickhouse.py @@ -22,7 +22,8 @@ TLoadJobState, FollowupJob, LoadJob, - NewLoadJob, StorageSchemaInfo, + NewLoadJob, + StorageSchemaInfo, ) from dlt.common.schema import Schema, TColumnSchema from dlt.common.schema.typing import ( @@ -386,11 +387,9 @@ def get_storage_table(self, table_name: str) -> Tuple[bool, TTableSchemaColumns] schema_table[c[0]] = schema_c # type: ignore return True, schema_table - def get_stored_schema(self) -> StorageSchemaInfo: return super().get_stored_schema() - @staticmethod def _gen_not_null(v: bool) -> str: # ClickHouse fields are not nullable by default. diff --git a/dlt/destinations/impl/clickhouse/configuration.py b/dlt/destinations/impl/clickhouse/configuration.py index 88b01d1cc8..562e0ca2e4 100644 --- a/dlt/destinations/impl/clickhouse/configuration.py +++ b/dlt/destinations/impl/clickhouse/configuration.py @@ -34,6 +34,8 @@ class ClickHouseCredentials(ConnectionStringCredentials): """Timeout for establishing connection. Defaults to 10 seconds.""" send_receive_timeout: int = 300 """Timeout for sending and receiving data. Defaults to 300 seconds.""" + dataset_table_separator: str = "___" + """Separator for dataset table names, defaults to '___', i.e. 'database.dataset___table'.""" __config_gen_annotations__: ClassVar[List[str]] = [ "host", @@ -44,6 +46,7 @@ class ClickHouseCredentials(ConnectionStringCredentials): "secure", "connect_timeout", "send_receive_timeout", + "dataset_table_separator", ] def parse_native_representation(self, native_value: Any) -> None: @@ -72,7 +75,7 @@ def to_url(self) -> URL: class ClickHouseClientConfiguration(DestinationClientDwhWithStagingConfiguration): destination_type: Final[str] = "clickhouse" # type: ignore[misc] credentials: ClickHouseCredentials # type: ignore - dataset_name: Final[str] = "" # type: ignore + dataset_name: Final[str] = "dlt" # type: ignore """dataset name in the destination to load data to, for schemas that are not default schema, it is used as dataset prefix""" # Primary key columns are used to build a sparse primary index which allows for efficient data retrieval, diff --git a/dlt/destinations/impl/clickhouse/sql_client.py b/dlt/destinations/impl/clickhouse/sql_client.py index 0af8933ae7..fd3c9a401b 100644 --- a/dlt/destinations/impl/clickhouse/sql_client.py +++ b/dlt/destinations/impl/clickhouse/sql_client.py @@ -53,6 +53,9 @@ def __init__(self, dataset_name: str, credentials: ClickHouseCredentials) -> Non self.credentials = credentials self.database_name = credentials.database + def has_dataset(self) -> bool: + return super().has_dataset() + def open_connection(self) -> clickhouse_driver.dbapi.connection.Connection: self._conn = clickhouse_driver.dbapi.connect( dsn=self.credentials.to_native_representation() @@ -157,9 +160,7 @@ def fully_qualified_dataset_name(self, escape: bool = True) -> str: return f"{database_name}.{dataset_name}" def make_qualified_table_name(self, table_name: str, escape: bool = True) -> str: - dataset_table_separator = dlt.config[ - "destination.clickhouse.credentials.dataset_table_separator" - ] + dataset_table_separator = self.credentials.dataset_table_separator if escape: database_name = self.capabilities.escape_identifier(self.database_name) dataset_and_table = self.capabilities.escape_identifier(