From 3c89b3bbc8284d2cbd794b8dc24d0de0bfd2f32f Mon Sep 17 00:00:00 2001 From: Marcel Coetzee Date: Wed, 10 Apr 2024 00:18:26 +0200 Subject: [PATCH] Set compression method to 'auto' for s3 table function #1055 Signed-off-by: Marcel Coetzee --- dlt/destinations/impl/clickhouse/clickhouse.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dlt/destinations/impl/clickhouse/clickhouse.py b/dlt/destinations/impl/clickhouse/clickhouse.py index 6cf3d9b49f..c4e7b9c02e 100644 --- a/dlt/destinations/impl/clickhouse/clickhouse.py +++ b/dlt/destinations/impl/clickhouse/clickhouse.py @@ -194,10 +194,11 @@ def __init__( secret_access_key = None clickhouse_format = FILE_FORMAT_TO_TABLE_FUNCTION_MAPPING[file_extension] + structure = "auto" template = Template(""" SELECT * FROM s3('{{ url }}'{% if access_key_id and secret_access_key %}, - '{{ access_key_id }}','{{ secret_access_key }}'{% else %},NOSIGN{% endif %},'{{ clickhouse_format }}') + '{{ access_key_id }}','{{ secret_access_key }}'{% else %},NOSIGN{% endif %},'{{ clickhouse_format }}','{{ structure }}','{{ compression }}') """) table_function = template.render( @@ -205,6 +206,8 @@ def __init__( access_key_id=access_key_id, secret_access_key=secret_access_key, clickhouse_format=clickhouse_format, + structure=structure, + compression=compression, ).strip() statement = f"INSERT INTO {qualified_table_name} {table_function}"