From 335fa289e4a07d60de5fcf3cbd290ba31615da83 Mon Sep 17 00:00:00 2001 From: Fokko Date: Wed, 15 Jan 2025 21:47:17 +0100 Subject: [PATCH] ADLS: Support Vended Credentials First version with `Fsspec`. Will check with PyArrow tomorrow, but that one seems to be lacking the SAS token: https://arrow.apache.org/docs/cpp/api/filesystem.html#azure-filesystem Closes #1146 --- pyiceberg/io/fsspec.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 62e9b9234..11faa3c97 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -179,6 +179,17 @@ def _gs(properties: Properties) -> AbstractFileSystem: def _adls(properties: Properties) -> AbstractFileSystem: from adlfs import AzureBlobFileSystem + for key, sas_token in { + key.replace(f"{ADLS_SAS_TOKEN}.", ""): value + for key, value in properties.items() + if key.startswith(ADLS_SAS_TOKEN) and key.endswith(".windows.net") + }.items(): + if key: + if ADLS_ACCOUNT_NAME not in properties: + properties[ADLS_ACCOUNT_NAME] = key.split(".")[0] + if ADLS_SAS_TOKEN not in properties: + properties[ADLS_SAS_TOKEN] = sas_token + return AzureBlobFileSystem( connection_string=properties.get(ADLS_CONNECTION_STRING), account_name=properties.get(ADLS_ACCOUNT_NAME),