Skip to content

Commit

Permalink
Set compression method to 'auto' for s3 table function #1055
Browse files Browse the repository at this point in the history
Signed-off-by: Marcel Coetzee <[email protected]>
  • Loading branch information
Pipboyguy committed Apr 9, 2024
1 parent e30ce6d commit 3c89b3b
Showing 1 changed file with 4 additions and 1 deletion.
5 changes: 4 additions & 1 deletion dlt/destinations/impl/clickhouse/clickhouse.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,17 +194,20 @@ def __init__(
secret_access_key = None

clickhouse_format = FILE_FORMAT_TO_TABLE_FUNCTION_MAPPING[file_extension]
structure = "auto"

template = Template("""
SELECT * FROM s3('{{ url }}'{% if access_key_id and secret_access_key %},
'{{ access_key_id }}','{{ secret_access_key }}'{% else %},NOSIGN{% endif %},'{{ clickhouse_format }}')
'{{ access_key_id }}','{{ secret_access_key }}'{% else %},NOSIGN{% endif %},'{{ clickhouse_format }}','{{ structure }}','{{ compression }}')
""")

table_function = template.render(
url=bucket_http_url,
access_key_id=access_key_id,
secret_access_key=secret_access_key,
clickhouse_format=clickhouse_format,
structure=structure,
compression=compression,
).strip()
statement = f"INSERT INTO {qualified_table_name} {table_function}"

Expand Down

0 comments on commit 3c89b3b

Please sign in to comment.