feat: introduce HFPath

megvii-research · Nov 20, 2024 · 20760de · 20760de
1 parent ac08679
commit 20760de
Show file tree

Hide file tree

Showing 4 changed files with 587 additions and 0 deletions.
diff --git a/megfile/__init__.py b/megfile/__init__.py
@@ -65,6 +65,7 @@
     is_hdfs,
 )
 from megfile.hdfs_path import HdfsPath
+from megfile.hf_path import HFPath
 from megfile.http import (
     http_exists,
     http_getmtime,
@@ -396,6 +397,7 @@
     "hdfs_makedirs",
     "S3Path",
     "FSPath",
+    "HFPath",
     "HttpPath",
     "HttpsPath",
     "StdioPath",

diff --git a/megfile/config.py b/megfile/config.py
@@ -60,5 +60,8 @@
 SFTP_MAX_RETRY_TIMES = int(
     os.getenv("MEGFILE_SFTP_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
 )
+HF_MAX_RETRY_TIMES = int(
+    os.getenv("MEGFILE_HF_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
+)
 
 HTTP_AUTH_HEADERS = ("Authorization", "Www-Authenticate", "Cookie", "Cookie2")