Skip to content

Commit

Permalink
Move ensure_branch out of _warn_on_[...] check
Browse files Browse the repository at this point in the history
  • Loading branch information
maxmynter committed Oct 16, 2023
1 parent 2c027ff commit ae53bce
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 10 deletions.
17 changes: 10 additions & 7 deletions src/lakefs_spec/spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import os
import urllib.error
import urllib.request
import warnings
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Generator
Expand Down Expand Up @@ -618,13 +619,15 @@ def __init__(
)

global _warn_on_fileupload
if mode == "wb" and _warn_on_fileupload:
logger.warning(
f"Calling `{self.__class__.__name__}.open()` in write mode results in unbuffered "
"file uploads, because the lakeFS Python client does not support multipart "
"uploads. Uploading large files unbuffered can have performance implications."
)
_warn_on_fileupload = False
if mode == "wb":
if _warn_on_fileupload:
warnings.warn(
f"Calling `{self.__class__.__name__}.open()` in write mode results in unbuffered "
"file uploads, because the lakeFS Python client does not support multipart "
"uploads. Uploading large files unbuffered can have performance implications.",
UserWarning,
)
_warn_on_fileupload = False
repository, branch, resource = parse(path)
ensure_branch(self.fs.client, repository, branch, self.fs.source_branch)

Expand Down
10 changes: 7 additions & 3 deletions tests/test_lakefs_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,13 @@ def test_lakefs_file_open_write(

rpath = f"{repository}/{temp_branch}/{random_file.name}"

# try opening the remote file and writing to it
with fs.open(rpath, "wb") as fp:
fp.write(orig_text)
with pytest.warns(
UserWarning,
match=r"Calling `LakeFSFile\.open\(\)` in write mode results in unbuffered file uploads, because the lakeFS Python client does not support multipart uploads\. Uploading large files unbuffered can have performance implications\.",
):
# try opening the remote file and writing to it
with fs.open(rpath, "wb") as fp:
fp.write(orig_text)

# pulling the written file down again, using ONLY built-in open (!)
lpath = random_file.with_name(random_file.name + "_copy")
Expand Down

0 comments on commit ae53bce

Please sign in to comment.