Skip to content

Commit

Permalink
ADLS: Support Vended Credentials
Browse files Browse the repository at this point in the history
First version with `Fsspec`. Will check with PyArrow tomorrow,
but that one seems to be lacking the SAS token:

https://arrow.apache.org/docs/cpp/api/filesystem.html#azure-filesystem

Closes #1146
  • Loading branch information
Fokko committed Jan 15, 2025
1 parent b806cfa commit 335fa28
Showing 1 changed file with 11 additions and 0 deletions.
11 changes: 11 additions & 0 deletions pyiceberg/io/fsspec.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,17 @@ def _gs(properties: Properties) -> AbstractFileSystem:
def _adls(properties: Properties) -> AbstractFileSystem:
from adlfs import AzureBlobFileSystem

for key, sas_token in {
key.replace(f"{ADLS_SAS_TOKEN}.", ""): value
for key, value in properties.items()
if key.startswith(ADLS_SAS_TOKEN) and key.endswith(".windows.net")
}.items():
if key:
if ADLS_ACCOUNT_NAME not in properties:
properties[ADLS_ACCOUNT_NAME] = key.split(".")[0]
if ADLS_SAS_TOKEN not in properties:
properties[ADLS_SAS_TOKEN] = sas_token

return AzureBlobFileSystem(
connection_string=properties.get(ADLS_CONNECTION_STRING),
account_name=properties.get(ADLS_ACCOUNT_NAME),
Expand Down

0 comments on commit 335fa28

Please sign in to comment.