Liveview sub directory support (#1974)

docs/release_notes/next/feature-1912-live-subdir

@@ -0,0 +1 @@
+#1912 : Support sub-directories in Live viewer

mantidimaging/gui/windows/live_viewer/model.py

@@ -2,6 +2,7 @@
 # SPDX - License - Identifier: GPL-3.0-or-later
 from __future__ import annotations
 
+import time
 from typing import TYPE_CHECKING
 from pathlib import Path
 from logging import getLogger
@@ -55,6 +56,18 @@ def image_modified_time(self) -> float:
         return self._stat.st_mtime
 
 
+class SubDirectory:
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self._stat = path.stat()
+        self.mtime = self._stat.st_mtime
+
+    @property
+    def modification_time(self) -> float:
+        return self.mtime
+
+
 class LiveViewerWindowModel:
     """
     The model for the spectrum viewer window.
@@ -96,7 +109,7 @@ def path(self, path: Path) -> None:
         self.image_watcher = ImageWatcher(path)
         self.image_watcher.image_changed.connect(self._handle_image_changed_in_list)
         self.image_watcher.recent_image_changed.connect(self.handle_image_modified)
-        self.image_watcher._handle_directory_change("")
+        self.image_watcher._handle_directory_change(str(path))
 
     def _handle_image_changed_in_list(self, image_files: list[Image_Data]) -> None:
         """
@@ -159,17 +172,19 @@ def __init__(self, directory: Path):
         self.directory = directory
         self.watcher = QFileSystemWatcher()
         self.watcher.directoryChanged.connect(self._handle_directory_change)
-        self.watcher.addPath(str(self.directory))
 
         self.recent_file_watcher = QFileSystemWatcher()
         self.recent_file_watcher.fileChanged.connect(self.handle_image_modified)
 
-    def find_images(self) -> list[Image_Data]:
+        self.sub_directories: dict[Path, SubDirectory] = {}
+        self.add_sub_directory(SubDirectory(self.directory))
+
+    def find_images(self, directory: Path) -> list[Image_Data]:
         """
         Find all the images in the directory.
         """
         image_files = []
-        for file_path in Path(self.directory).iterdir():
+        for file_path in directory.iterdir():
             if self._is_image_file(file_path.name):
                 try:
                     image_obj = Image_Data(file_path)
@@ -179,6 +194,19 @@ def find_images(self) -> list[Image_Data]:
 
         return image_files
 
+    def find_sub_directories(self, directory: Path) -> None:
+        # COMPAT python < 3.12 - Can replace with Path.walk()
+        try:
+            for filename in directory.glob("**/*"):
+                if filename.is_dir():
+                    self.add_sub_directory(SubDirectory(filename))
+        except FileNotFoundError:
+            pass
+
+    def sort_sub_directory_by_modified_time(self) -> None:
+        self.sub_directories = dict(
+            sorted(self.sub_directories.items(), key=lambda p: p[1].modification_time, reverse=True))
+
     @staticmethod
     def sort_images_by_modified_time(images: list[Image_Data]) -> list[Image_Data]:
         """
@@ -197,8 +225,28 @@ def _handle_directory_change(self, directory: str) -> None:
 
         :param directory: directory that has changed
         """
+        directory_path = Path(directory)
+
+        # Force the modification time of signal directory, because file changes may not update
+        # parent dir mtime
+        if directory_path.exists():
+            this_dir = SubDirectory(directory_path)
+            this_dir.mtime = time.time()
+            self.add_sub_directory(this_dir)
+
+        self.clear_deleted_sub_directories(directory_path)
+        self.find_sub_directories(directory_path)
+        self.sort_sub_directory_by_modified_time()
+
+        for newest_directory in self.sub_directories.values():
+            try:
+                images = self.find_images(newest_directory.path)
+            except FileNotFoundError:
+                images = []
+
+            if len(images) > 0:
+                break
 
-        images = self.find_images()
         images = self.sort_images_by_modified_time(images)
         self.update_recent_watcher(images[-1:])
         self.image_changed.emit(images)
@@ -219,7 +267,7 @@ def remove_path(self):
         """
         Remove the currently set path
         """
-        self.watcher.removePath(str(self.directory))
+        self.watcher.removePaths([str(path) for path in self.sub_directories.keys()])
         self.recent_file_watcher.removePaths(self.recent_file_watcher.files())
         assert len(self.watcher.files()) == 0
         assert len(self.watcher.directories()) == 0
@@ -232,3 +280,20 @@ def update_recent_watcher(self, images: list[Image_Data]) -> None:
 
     def handle_image_modified(self, file_path):
         self.recent_image_changed.emit(Path(file_path))
+
+    def add_sub_directory(self, sub_dir: SubDirectory):
+        if sub_dir.path not in self.sub_directories:
+            self.watcher.addPath(str(sub_dir.path))
+
+        self.sub_directories[sub_dir.path] = sub_dir
+
+    def remove_sub_directory(self, sub_dir: Path):
+        if sub_dir in self.sub_directories:
+            self.watcher.removePath(str(sub_dir))
+
+        del self.sub_directories[sub_dir]
+
+    def clear_deleted_sub_directories(self, directory: Path):
+        for sub_dir in list(self.sub_directories):
+            if sub_dir.is_relative_to(directory) and not sub_dir.exists():
+                self.remove_sub_directory(sub_dir)

mantidimaging/gui/windows/live_viewer/test/model_test.py

@@ -0,0 +1,141 @@
+# Copyright (C) 2023 ISIS Rutherford Appleton Laboratory UKRI
+# SPDX - License - Identifier: GPL-3.0-or-later
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+from unittest import mock
+
+from PyQt5.QtCore import QFileSystemWatcher, pyqtSignal
+
+from mantidimaging.gui.windows.live_viewer.model import ImageWatcher
+from mantidimaging.test_helpers.unit_test_helper import FakeFSTestCase
+
+
+class ImageWatcherTest(FakeFSTestCase):
+
+    def setUp(self) -> None:
+        super().setUp()
+        self.top_path = Path("/live")
+        self.fs.create_dir(self.top_path)
+        os.utime(self.top_path, (10, 10))
+        with mock.patch("mantidimaging.gui.windows.live_viewer.model.QFileSystemWatcher") as mocker:
+            mock_dir_watcher = mock.create_autospec(QFileSystemWatcher, directoryChanged=mock.Mock())
+            mock_file_watcher = mock.create_autospec(QFileSystemWatcher, fileChanged=mock.Mock())
+
+            mocker.side_effect = [mock_dir_watcher, mock_file_watcher]
+            self.watcher = ImageWatcher(self.top_path)
+            self.mock_signal_image = mock.create_autospec(pyqtSignal, emit=mock.Mock())
+            self.watcher.image_changed = self.mock_signal_image
+
+    def _make_simple_dir(self, directory: Path, t0: float = 1000):
+        file_list = [directory / f"abc_{i:06d}.tif" for i in range(5)]
+        if not directory.exists():
+            self.fs.create_dir(directory)
+        os.utime(directory, (10, t0))
+        n = 1
+        for file in file_list:
+            self.fs.create_file(file)
+            os.utime(file, (10, t0 + n))
+            n += 1
+
+        return file_list
+
+    def _make_sub_directories(self, directory: Path, sub_dirs: list[str], t0: float = 1000):
+        n = 0
+        for sub_dir in sub_dirs:
+            self.fs.create_dir(directory / sub_dir)
+            os.utime(directory / sub_dir, (10, t0 + n))
+            n += 1
+            file_list = [directory / sub_dir / f"abc_{i:06d}.tiff" for i in range(5)]
+            for file in file_list:
+                self.fs.create_file(file)
+                os.utime(file, (10, t0 + n))
+                n += 1
+
+        return file_list
+
+    def _get_recent_emitted_files(self):
+        self.mock_signal_image.emit.assert_called()
+        last_call_args = self.mock_signal_image.emit.call_args_list[-1]
+        emitted_images = [image_data.image_path for image_data in last_call_args[0][0]]
+        return emitted_images
+
+    def test_WHEN_find_images_called_THEN_returns_images(self):
+        file_list = self._make_simple_dir(self.top_path)
+
+        images_datas = self.watcher.find_images(self.top_path)
+
+        images = [image.image_path for image in images_datas]
+        self._file_list_count_equal(images, file_list)
+
+    def test_WHEN_find_images_deleted_file_THEN_handles_error(self):
+        # Simulate the case where a file returned by iterdir has been deleted when we come to read it
+        file_list = self._make_simple_dir(self.top_path)
+        self.fs.remove(file_list[0])
+
+        # mocked iterdir() gives full list, but first file as been deleted
+        mock_directory = mock.create_autospec(Path, iterdir=lambda: file_list)
+
+        images_datas = self.watcher.find_images(mock_directory)
+
+        images = [image.image_path for image in images_datas]
+        self._file_list_count_equal(images, file_list[1:])
+
+    def test_WHEN_find_sub_directories_called_THEN_finds_subdirs(self):
+        self._file_in_sequence(self.top_path, self.watcher.sub_directories.keys())
+        self.assertEqual(len(self.watcher.sub_directories), 1)
+        subdir_names = ["tomo", "flat_before", "flat_after"]
+        self._make_sub_directories(self.top_path, subdir_names)
+
+        self.watcher.find_sub_directories(self.top_path)
+        for sub_dir in subdir_names:
+            self._file_in_sequence(self.top_path / sub_dir, self.watcher.sub_directories.keys())
+
+    def test_WHEN_directory_change_simple_THEN_images_emitted(self):
+        file_list = self._make_simple_dir(self.top_path)
+        self.mock_signal_image.emit.assert_not_called()
+
+        self.watcher._handle_directory_change(self.top_path)
+
+        emitted_images = self._get_recent_emitted_files()
+        self._file_list_count_equal(emitted_images, file_list)
+
+    @mock.patch("time.time", return_value=4000.0)
+    def test_WHEN_directory_change_empty_subdir_THEN_images_emitted(self, _mock_time):
+        # empty sub dir created, but contains no images so should emit images from top dir
+        file_list = self._make_simple_dir(self.top_path)
+        self.fs.create_dir(self.top_path / "empty")
+        os.utime(self.top_path / "empty", [10, 2000])
+        self.assertLess(self.top_path.stat().st_mtime, (self.top_path / 'empty').stat().st_mtime)
+
+        self.watcher._handle_directory_change(self.top_path / "empty")
+
+        emitted_images = self._get_recent_emitted_files()
+        self._file_list_count_equal(emitted_images, file_list)
+
+    @mock.patch("time.time", return_value=4000.0)
+    def test_WHEN_directory_change_with_subdir_THEN_images_emitted(self, _mock_time):
+        # If change notification is on top dir, then emit top files even if subdir newer
+        file_list = self._make_simple_dir(self.top_path)
+        _ = self._make_simple_dir(self.top_path / "more", t0=2000)
+        self.assertLess(self.top_path.stat().st_mtime, (self.top_path / 'more').stat().st_mtime)
+        self.assertLess((self.top_path / 'more').stat().st_mtime, time.time())
+
+        self.watcher._handle_directory_change(self.top_path)
+
+        emitted_images = self._get_recent_emitted_files()
+        self._file_list_count_equal(emitted_images, file_list)
+
+    @mock.patch("time.time", return_value=4000.0)
+    def test_WHEN_sub_directory_change_THEN_images_emitted(self, _mock_time):
+        # If change notification is on sub dir, then emit sub dir files
+        _ = self._make_simple_dir(self.top_path)
+        file_list2 = self._make_simple_dir(self.top_path / "more", t0=2000)
+        self.assertLess(self.top_path.stat().st_mtime, (self.top_path / 'more').stat().st_mtime)
+
+        self.watcher._handle_directory_change(self.top_path / "more")
+
+        emitted_images = self._get_recent_emitted_files()
+        self._file_list_count_equal(emitted_images, file_list2)