Skip to content

Commit

Permalink
fix databricks volume file name
Browse files Browse the repository at this point in the history
  • Loading branch information
donotpush committed Jan 20, 2025
1 parent e772d20 commit 2bd0be0
Showing 1 changed file with 6 additions and 4 deletions.
10 changes: 6 additions & 4 deletions dlt/destinations/impl/databricks/databricks.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
from dlt.destinations.sql_jobs import SqlMergeFollowupJob
from dlt.destinations.job_impl import ReferenceFollowupJobRequest
from dlt.destinations.utils import is_compression_disabled
from dlt.common.utils import digest128
from dlt.common.utils import uniq_id

SUPPORTED_BLOB_STORAGE_PROTOCOLS = AZURE_BLOB_STORAGE_PROTOCOLS + S3_PROTOCOLS + GCS_PROTOCOLS

Expand Down Expand Up @@ -118,10 +118,12 @@ def _handle_local_file_upload(self, local_file_path: str) -> tuple[str, str]:
elif file_name.endswith(".jsonl"):
file_format = "jsonl"
else:
return "",file_name
return "", file_name

volume_path = f"/Volumes/{self._sql_client.database_name}/{self._sql_client.dataset_name}/{self._sql_client.volume_name}/{time.time_ns()}"
volume_file_name = f"{digest128(file_name)}.{file_format}" # file_name must be hashed - databricks fails with file name starting with - or .
volume_file_name = ( # replace file_name for random hex code - databricks loading fails when file_name starts with - or .
f"{uniq_id()}.{file_format}"
)
volume_file_path = f"{volume_path}/{volume_file_name}"

with open(local_file_path, "rb") as f:
Expand Down

0 comments on commit 2bd0be0

Please sign in to comment.