Merge pull request #4 from neuro-ml/dev

Fixed file permissions for cache index

tarn/__version__.py

@@ -1 +1 @@
-__version__ = '0.0.4'
+__version__ = '0.0.5'

tarn/cache/index.py

@@ -3,13 +3,14 @@
 import os
 import shutil
 import warnings
+from itertools import chain
 from pathlib import Path
 from typing import Any
 
 from ..digest import key_to_relative
 from ..local import Storage, DiskBase
 from ..interface import Key
-from ..utils import create_folders, to_read_only, copy_file, match_files
+from ..utils import create_folders, to_read_only, copy_file, match_files, adjust_permissions
 from ..exceptions import StorageCorruption, ReadError
 from .serializers import Serializer, SerializerError
 from .compat import BadGzipFile
@@ -45,7 +46,11 @@ def _write(self, base: Path, key: Key, value: Any, context: Any):
 
     def _replicate(self, base: Path, key: Key, source: Path, context):
         # data
-        shutil.copytree(source / DATA_FOLDER, base / DATA_FOLDER)
+        destination = base / DATA_FOLDER
+        shutil.copytree(source / DATA_FOLDER, destination)
+        for dst in chain([destination], destination.rglob('*')):
+            adjust_permissions(dst, self.permissions, self.group, read_only=not dst.is_dir())
+
         # meta
         copy_file(source / HASH_FILENAME, base / HASH_FILENAME)
         to_read_only(base / HASH_FILENAME, self.permissions, self.group)
@@ -99,7 +104,7 @@ def _mirror_to_storage(self, source: Path, destination: Path):
         for file in source.glob('**/*'):
             target = destination / file.relative_to(source)
             if file.is_dir():
-                target.mkdir(parents=True)
+                create_folders(target, self.permissions, self.group)
 
             else:
                 with open(target, 'w') as fd:

tarn/ssh.py

@@ -70,10 +70,14 @@ def fetch(self, keys: Sequence[Key], store: Callable[[Key, Path], Any],
                 for key in keys:
                     try:
                         scp.get(str(self.root / key_to_relative(key, self.levels)), str(source), recursive=True)
-                        value = store(key, source)
-                        shutil.rmtree(source)
+                        if not source.exists():
+                            results.append((None, False))
+
+                        else:
+                            value = store(key, source)
+                            shutil.rmtree(source)
+                            results.append((value, True))
 
-                        results.append((value, True))
                     except (SCPException, socket.timeout):
                         results.append((None, False))
 

tarn/utils.py

@@ -15,7 +15,14 @@ class Reason(Enum):
 
 
 def to_read_only(path: Path, permissions, group):
-    os.chmod(path, 0o444 & permissions)
+    adjust_permissions(path, permissions, group, read_only=True)
+
+
+def adjust_permissions(path: Path, permissions, group, read_only: bool = False):
+    if read_only:
+        permissions = 0o444 & permissions
+
+    os.chmod(path, permissions)
     shutil.chown(path, group=group)