From ab86362e4f5ce08dd70aa1842a2e94d147e11d3f Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 20 Feb 2024 17:05:46 +0100 Subject: [PATCH 01/21] include lzf --- kwave/options/simulation_options.py | 10 ++++++---- kwave/utils/io.py | 4 ++-- .../python_testers/h5io_test.py | 5 +++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/kwave/options/simulation_options.py b/kwave/options/simulation_options.py index d1a7d0df..372ee320 100644 --- a/kwave/options/simulation_options.py +++ b/kwave/options/simulation_options.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field from enum import Enum from tempfile import gettempdir -from typing import List, Optional, TYPE_CHECKING +from typing import List, Union, Optional, TYPE_CHECKING import numpy as np @@ -102,7 +102,7 @@ class SimulationOptions(object): stream_to_disk: bool = False data_recast: Optional[bool] = False cartesian_interp: str = 'linear' - hdf_compression_level: Optional[int] = None + hdf_compression_level: Optional[Union[int, str]] = None data_cast: str = 'off' pml_search_range: List[int] = field(default_factory=lambda: [10, 40]) radial_symmetry: str = 'WSWA-FFT' @@ -134,8 +134,10 @@ def __post_init__(self): h5_literals = get_h5_literals() self.hdf_compression_level = h5_literals.HDF_COMPRESSION_LEVEL # check value is an integer between 0 and 9 - assert isinstance(self.hdf_compression_level, int) and 0 <= self.hdf_compression_level <= 9, \ - "Optional input ''hdf_compression_level'' must be an integer between 0 and 9." + assert ((isinstance(self.hdf_compression_level, int) and (0 <= self.hdf_compression_level <= 9)) or + (isinstance(self.hdf_compression_level, str) and ((self.hdf_compression_level.lower() == 'lzf') or + (self.hdf_compression_level.lower() == 'szip')))), \ + "Optional input ''hdf_compression_level'' is false: must be between 0-9 or either lzf or szip" assert np.isscalar(self.multi_axial_PML_ratio) and self.multi_axial_PML_ratio >= 0, \ "Optional input ''multi_axial_PML_ratio'' must be a single positive value." diff --git a/kwave/utils/io.py b/kwave/utils/io.py index da9f8237..08f46e25 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -46,8 +46,8 @@ def get_h5_literals(): 'HDF_FILE_MAJOR_VERSION': '1', 'HDF_FILE_MINOR_VERSION': '2', - # compression level - 'HDF_COMPRESSION_LEVEL': 0 + # compression level: set to be same as default h5py + 'HDF_COMPRESSION_LEVEL': 4 }) return literals diff --git a/tests/matlab_test_data_collectors/python_testers/h5io_test.py b/tests/matlab_test_data_collectors/python_testers/h5io_test.py index a41925f8..7be99a63 100644 --- a/tests/matlab_test_data_collectors/python_testers/h5io_test.py +++ b/tests/matlab_test_data_collectors/python_testers/h5io_test.py @@ -24,12 +24,13 @@ def compare_h5_values(local_h5_path, ref_path): def test_write_matrix(tmp_path_factory): + compression_options = [1,2,3,4,5,6,7,8,9,'lzf'] idx = 0 for dim in range(1, 3): - for compression_level in range(1, 9): + for compression_level in compression_options: tmp_path = tmp_path_factory.mktemp("matrix") / f"{idx}.h5" matrix = np.single(10.0 * np.ones([1, dim])) - write_matrix(tmp_path, matrix=matrix, matrix_name='test') + write_matrix(tmp_path, matrix=matrix, matrix_name='test', compression_level=compression_level) ref_path = os.path.join(Path(__file__).parent, f"collectedValues/writeMatrix/{idx}.h5") compare_h5_values(tmp_path, ref_path) idx = idx + 1 From 7b26565c36478f7e5e8843c14249f0f476ed4f71 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 20 Feb 2024 17:21:12 +0100 Subject: [PATCH 02/21] revert --- tests/matlab_test_data_collectors/python_testers/h5io_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/matlab_test_data_collectors/python_testers/h5io_test.py b/tests/matlab_test_data_collectors/python_testers/h5io_test.py index 7be99a63..8d70e918 100644 --- a/tests/matlab_test_data_collectors/python_testers/h5io_test.py +++ b/tests/matlab_test_data_collectors/python_testers/h5io_test.py @@ -30,7 +30,7 @@ def test_write_matrix(tmp_path_factory): for compression_level in compression_options: tmp_path = tmp_path_factory.mktemp("matrix") / f"{idx}.h5" matrix = np.single(10.0 * np.ones([1, dim])) - write_matrix(tmp_path, matrix=matrix, matrix_name='test', compression_level=compression_level) + write_matrix(tmp_path, matrix=matrix, matrix_name='test') ref_path = os.path.join(Path(__file__).parent, f"collectedValues/writeMatrix/{idx}.h5") compare_h5_values(tmp_path, ref_path) idx = idx + 1 From 9f1b32136b70f74a0238a6da13537153e0d72313 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 27 Feb 2024 10:34:19 +0100 Subject: [PATCH 03/21] Update simulation_options.py change name to hdf_compression_options --- kwave/options/simulation_options.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/kwave/options/simulation_options.py b/kwave/options/simulation_options.py index 372ee320..90328c3f 100644 --- a/kwave/options/simulation_options.py +++ b/kwave/options/simulation_options.py @@ -70,7 +70,7 @@ class SimulationOptions(object): The saved variables can be used to run simulations using the C++ code. data_recast: recast the sensor data back to double precision cartesian_interp: interpolation mode for Cartesian sensor mask - hdf_compression_level: zip compression level for HDF5 input files + hdf_compression_options: either gzip compression level for HDF5 input files, or type of compression used data_cast: data cast pml_search_range: search range used when automatically determining PML size radial_symmetry: radial symmetry used in axisymmetric code @@ -130,14 +130,14 @@ def __post_init__(self): if self.data_cast == 'double': self.data_cast = 'off' - # load the HDF5 literals (for the default compression level) + # load the HDF5 literals (for the default compression settings) h5_literals = get_h5_literals() - self.hdf_compression_level = h5_literals.HDF_COMPRESSION_LEVEL + self.hdf_compression_options = h5_literals.HDF_COMPRESSION_LEVEL # check value is an integer between 0 and 9 - assert ((isinstance(self.hdf_compression_level, int) and (0 <= self.hdf_compression_level <= 9)) or - (isinstance(self.hdf_compression_level, str) and ((self.hdf_compression_level.lower() == 'lzf') or - (self.hdf_compression_level.lower() == 'szip')))), \ - "Optional input ''hdf_compression_level'' is false: must be between 0-9 or either lzf or szip" + assert ((isinstance(self.hdf_compression_options, int) and (0 <= self.hdf_compression_options <= 9)) or + (isinstance(self.hdf_compression_options, str) and ((self.hdf_compression_options.lower() == 'lzf') or + (self.hdf_compression_options.lower() == 'szip')))), \ + "Optional input ''hdf_compression_options'' is false: must an integer be between 0-9 or either 'lzf' or 'szip'" assert np.isscalar(self.multi_axial_PML_ratio) and self.multi_axial_PML_ratio >= 0, \ "Optional input ''multi_axial_PML_ratio'' must be a single positive value." @@ -208,9 +208,11 @@ def option_factory(kgrid: "kWaveGrid", options: SimulationOptions): * data_recast: Boolean controlling whether the output data is cast back to double precision. If set to false, sensor_data will be returned in the data format set using the 'data_cast' option. - * hdf_compression_level: Compression level used for writing the input HDF5 file when using + * hdf_compression_options: Compression level used for writing the input HDF5 file when using 'save_to_disk' or kspaceFirstOrder3DC. Can be set to an integer - between 0 (no compression, the default) and 9 (maximum compression). + between 0 (no compression, the default) and 9 (maximum compression) for gzip + compression or as a string for lzf or szip compression. + Note that szip compression requires additional libraries to be installed. The compression is lossless. Increasing the compression level will reduce the file size if there are portions of the medium that are homogeneous, but will also increase the time to create the HDF5 file. From cda3757c6240ee5c4428f1f1720aa5cca857a782 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 27 Feb 2024 10:44:21 +0100 Subject: [PATCH 04/21] rename compression_level to compression_options --- kwave/utils/io.py | 18 +++++++++--------- .../python_testers/h5io_test.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/kwave/utils/io.py b/kwave/utils/io.py index 2b3f27fd..9dcc1db4 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -3,7 +3,7 @@ import platform import socket from datetime import datetime -from typing import Optional +from typing import Optional, Union import cv2 import h5py @@ -47,19 +47,19 @@ def get_h5_literals(): 'HDF_FILE_MINOR_VERSION': '2', # compression level: set to be same as default h5py - 'HDF_COMPRESSION_LEVEL': 4 + 'HDF_COMPRESSION_OPTIONS': 4 }) return literals -def write_matrix(filename, matrix: np.ndarray, matrix_name: str, compression_level:int =None, auto_chunk: bool =True): +def write_matrix(filename, matrix: np.ndarray, matrix_name: str, compression_options: Union[int, str] = None, auto_chunk: bool = True): # get literals h5_literals = get_h5_literals() assert isinstance(auto_chunk, bool), "auto_chunk must be a boolean." - if compression_level is None: - compression_level = h5_literals.HDF_COMPRESSION_LEVEL + if compression_options is None: + compression_options = h5_literals.HDF_COMPRESSION_OPTIONS # dims = num_dim(matrix) dims = len(matrix.shape) @@ -78,7 +78,7 @@ def write_matrix(filename, matrix: np.ndarray, matrix_name: str, compression_lev else: Nx, Ny, Nz = 1, 1, 1 - # check size of matrix and set chunk size and compression level + # check size of matrix and set chunk size and compression options if dims == 3: # set chunk size to Nx * Ny chunk_size = [Nx, Ny, 1] @@ -99,7 +99,7 @@ def write_matrix(filename, matrix: np.ndarray, matrix_name: str, compression_lev else: # set no compression - compression_level = 0 + compression_options = 0 # set chunk size to grid size if matrix.size == 1: @@ -186,9 +186,9 @@ def write_matrix(filename, matrix: np.ndarray, matrix_name: str, compression_lev 'chunks': auto_chunk if auto_chunk is True else tuple(chunk_size) } - if compression_level != 0: + if compression_options != 0: # use compression - opts['compression'] = compression_level + opts['compression'] = compression_options # write the matrix into the file with h5py.File(filename, "a") as f: diff --git a/tests/matlab_test_data_collectors/python_testers/h5io_test.py b/tests/matlab_test_data_collectors/python_testers/h5io_test.py index 49342b0a..1d80634c 100644 --- a/tests/matlab_test_data_collectors/python_testers/h5io_test.py +++ b/tests/matlab_test_data_collectors/python_testers/h5io_test.py @@ -26,13 +26,13 @@ def compare_h5_values(local_h5_path, ref_path): def test_write_matrix(tmp_path_factory): - compression_options = [1,2,3,4,5,6,7,8,9,'lzf'] + compression_options = range(1,9) #[1,2,3,4,5,6,7,8,9,'lzf'] idx = 0 for dim in range(1, 3): for compression_level in compression_options: tmp_path = tmp_path_factory.mktemp("matrix") / f"{idx}.h5" matrix = np.single(10.0 * np.ones([1, dim])) - write_matrix(tmp_path, matrix=matrix, matrix_name='test') + write_matrix(tmp_path, matrix=matrix, matrix_name='test', compression_level=compression_level) ref_path = os.path.join(Path(__file__).parent, f"collectedValues/writeMatrix/{idx}.h5") compare_h5_values(tmp_path, ref_path) idx = idx + 1 From 0c06046f3dbb8b55057923c0b52b374fd5283eae Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 27 Feb 2024 10:51:11 +0100 Subject: [PATCH 05/21] renaming to compression_options --- kwave/kWaveSimulation_helper/save_to_disk_func.py | 12 ++++++------ .../python_testers/h5io_test.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/kwave/kWaveSimulation_helper/save_to_disk_func.py b/kwave/kWaveSimulation_helper/save_to_disk_func.py index 4faef74e..3a7f4e7c 100644 --- a/kwave/kWaveSimulation_helper/save_to_disk_func.py +++ b/kwave/kWaveSimulation_helper/save_to_disk_func.py @@ -57,7 +57,7 @@ def save_to_disk_func( # ========================================================================= remove_z_dimension(float_variables, kgrid.dim) - save_file(opt.input_filename, integer_variables, float_variables, opt.hdf_compression_level, + save_file(opt.input_filename, integer_variables, float_variables, opt.hdf_compression_options, auto_chunk=auto_chunk) # update command line status @@ -447,12 +447,12 @@ def enforce_filename_standards(filepath): return filepath, filename_ext -def save_file(filepath, integer_variables, float_variables, hdf_compression_level, auto_chunk): +def save_file(filepath, integer_variables, float_variables, hdf_compression_options, auto_chunk): filepath, filename_ext = enforce_filename_standards(filepath) # save file if filename_ext == '.h5': - save_h5_file(filepath, integer_variables, float_variables, hdf_compression_level, auto_chunk) + save_h5_file(filepath, integer_variables, float_variables, hdf_compression_options, auto_chunk) elif filename_ext == '.mat': save_mat_file(filepath, integer_variables, float_variables) @@ -461,7 +461,7 @@ def save_file(filepath, integer_variables, float_variables, hdf_compression_leve raise NotImplementedError('unknown file extension for ''save_to_disk'' filename') -def save_h5_file(filepath, integer_variables, float_variables, hdf_compression_level, auto_chunk): +def save_h5_file(filepath, integer_variables, float_variables, hdf_compression_options, auto_chunk): # ---------------- # SAVE HDF5 FILE # ---------------- @@ -476,7 +476,7 @@ def save_h5_file(filepath, integer_variables, float_variables, hdf_compression_l for key, value in float_variables.items(): # cast matrix to single precision value = np.array(value, dtype=np.float32) - write_matrix(filepath, value, key, hdf_compression_level, auto_chunk) + write_matrix(filepath, value, key, hdf_compression_options, auto_chunk) del value # change all the index variables to be in 64-bit unsigned integers @@ -484,7 +484,7 @@ def save_h5_file(filepath, integer_variables, float_variables, hdf_compression_l for key, value in integer_variables.items(): # cast matrix to 64-bit unsigned integer value = np.array(value, dtype=np.uint64) - write_matrix(filepath, value, key, hdf_compression_level, auto_chunk) + write_matrix(filepath, value, key, hdf_compression_options, auto_chunk) del value # set additional file attributes diff --git a/tests/matlab_test_data_collectors/python_testers/h5io_test.py b/tests/matlab_test_data_collectors/python_testers/h5io_test.py index 1d80634c..c92ca73b 100644 --- a/tests/matlab_test_data_collectors/python_testers/h5io_test.py +++ b/tests/matlab_test_data_collectors/python_testers/h5io_test.py @@ -29,10 +29,10 @@ def test_write_matrix(tmp_path_factory): compression_options = range(1,9) #[1,2,3,4,5,6,7,8,9,'lzf'] idx = 0 for dim in range(1, 3): - for compression_level in compression_options: + for compression_option in compression_options: tmp_path = tmp_path_factory.mktemp("matrix") / f"{idx}.h5" matrix = np.single(10.0 * np.ones([1, dim])) - write_matrix(tmp_path, matrix=matrix, matrix_name='test', compression_level=compression_level) + write_matrix(tmp_path, matrix=matrix, matrix_name='test', compression_options=compression_option) ref_path = os.path.join(Path(__file__).parent, f"collectedValues/writeMatrix/{idx}.h5") compare_h5_values(tmp_path, ref_path) idx = idx + 1 From a68555d450353fa7215849f9e38182d556a96928 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 27 Feb 2024 10:58:53 +0100 Subject: [PATCH 06/21] another renaming --- kwave/options/simulation_options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kwave/options/simulation_options.py b/kwave/options/simulation_options.py index 90328c3f..38741b99 100644 --- a/kwave/options/simulation_options.py +++ b/kwave/options/simulation_options.py @@ -102,7 +102,7 @@ class SimulationOptions(object): stream_to_disk: bool = False data_recast: Optional[bool] = False cartesian_interp: str = 'linear' - hdf_compression_level: Optional[Union[int, str]] = None + hdf_compression_options: Optional[Union[int, str]] = None data_cast: str = 'off' pml_search_range: List[int] = field(default_factory=lambda: [10, 40]) radial_symmetry: str = 'WSWA-FFT' @@ -132,7 +132,7 @@ def __post_init__(self): # load the HDF5 literals (for the default compression settings) h5_literals = get_h5_literals() - self.hdf_compression_options = h5_literals.HDF_COMPRESSION_LEVEL + self.hdf_compression_options = h5_literals.HDF_COMPRESSION_OPTIONS # check value is an integer between 0 and 9 assert ((isinstance(self.hdf_compression_options, int) and (0 <= self.hdf_compression_options <= 9)) or (isinstance(self.hdf_compression_options, str) and ((self.hdf_compression_options.lower() == 'lzf') or From 5a5f8385bb7839341988dbb8b8009c8602dbab71 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 27 Feb 2024 11:21:56 +0100 Subject: [PATCH 07/21] new test --- .../python_testers/h5io_test.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/matlab_test_data_collectors/python_testers/h5io_test.py b/tests/matlab_test_data_collectors/python_testers/h5io_test.py index c92ca73b..551a3c4f 100644 --- a/tests/matlab_test_data_collectors/python_testers/h5io_test.py +++ b/tests/matlab_test_data_collectors/python_testers/h5io_test.py @@ -26,7 +26,7 @@ def compare_h5_values(local_h5_path, ref_path): def test_write_matrix(tmp_path_factory): - compression_options = range(1,9) #[1,2,3,4,5,6,7,8,9,'lzf'] + compression_options = range(1,9) idx = 0 for dim in range(1, 3): for compression_option in compression_options: @@ -86,3 +86,34 @@ def test_write_grid(tmp_path_factory): compare_h5_values(tmp_path, ref_path) idx = idx + 1 pass + + +def test_lzf(self): + """ Create with explicit lzf """ + dset = self.f.create_dataset('foo', (20, 30), compression='lzf') + self.assertEqual(dset.compression, 'lzf') + self.assertEqual(dset.compression_opts, None) + + testdata = np.arange(100) + dset = self.f.create_dataset('bar', data=testdata, compression='lzf') + self.assertEqual(dset.compression, 'lzf') + self.assertEqual(dset.compression_opts, None) + + self.f.flush() # Actually write to file + + readdata = self.f['bar'][()] + self.assertArrayEqual(readdata, testdata) + +def test_write_matrix_lzf(tmp_path_factory): + """ + Tests the compression option `lzf`, which is not an option for the matlab h5create function, by + comparing written data to a reference matrix + """ + compression_option = 'lzf' + for idx, dim in enumerate(range(2, 3)): + tmp_path = tmp_path_factory.mktemp("matrix") / f"{idx}.h5" + matrix = np.single(10.0 * np.ones([1, dim])) + write_matrix(tmp_path, matrix=matrix, matrix_name='test', compression_options=compression_option) + tmp_h5 = h5py.File(tmp_path, 'r') + assert np.isclose(tmp_h5['test'], matrix).all() + pass \ No newline at end of file From b93096ee5cb255abed331f77dffae2c4ca09a751 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 27 Feb 2024 11:30:14 +0100 Subject: [PATCH 08/21] remove irrelevant code --- .../python_testers/h5io_test.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/matlab_test_data_collectors/python_testers/h5io_test.py b/tests/matlab_test_data_collectors/python_testers/h5io_test.py index 551a3c4f..8e7e0c48 100644 --- a/tests/matlab_test_data_collectors/python_testers/h5io_test.py +++ b/tests/matlab_test_data_collectors/python_testers/h5io_test.py @@ -88,22 +88,6 @@ def test_write_grid(tmp_path_factory): pass -def test_lzf(self): - """ Create with explicit lzf """ - dset = self.f.create_dataset('foo', (20, 30), compression='lzf') - self.assertEqual(dset.compression, 'lzf') - self.assertEqual(dset.compression_opts, None) - - testdata = np.arange(100) - dset = self.f.create_dataset('bar', data=testdata, compression='lzf') - self.assertEqual(dset.compression, 'lzf') - self.assertEqual(dset.compression_opts, None) - - self.f.flush() # Actually write to file - - readdata = self.f['bar'][()] - self.assertArrayEqual(readdata, testdata) - def test_write_matrix_lzf(tmp_path_factory): """ Tests the compression option `lzf`, which is not an option for the matlab h5create function, by From 5d0caf3b9aa80c1f6555bca9e86f5421c07e1bc7 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 5 Mar 2024 17:05:05 +0100 Subject: [PATCH 09/21] first pass at enum --- kwave/options/simulation_options.py | 27 +++++++++++++++++++++++---- kwave/utils/io.py | 12 ++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/kwave/options/simulation_options.py b/kwave/options/simulation_options.py index 1d7d9711..5856b6d5 100644 --- a/kwave/options/simulation_options.py +++ b/kwave/options/simulation_options.py @@ -41,6 +41,25 @@ def is_axisymmetric(self): return self == SimulationType.AXISYMMETRIC +class CompressionOption(Enum): + """ + Enum for the compression options + """ + GZIP_0 = 0 + GZIP_1 = 1 + GZIP_2 = 2 + GZIP_3 = 3 + GZIP_4 = 4 + GZIP_5 = 5 + GZIP_6 = 6 + GZIP_7 = 7 + GZIP_8 = 8 + GZIP_9 = 9 + LZF = 'lzf' + SZIP = 'szip' + + + @dataclass class SimulationOptions(object): """ @@ -102,11 +121,11 @@ class SimulationOptions(object): stream_to_disk: bool = False data_recast: Optional[bool] = False cartesian_interp: str = 'linear' - hdf_compression_options: Optional[Union[int, str]] = None data_cast: str = 'off' pml_search_range: List[int] = field(default_factory=lambda: [10, 40]) radial_symmetry: str = 'WSWA-FFT' multi_axial_PML_ratio: float = 0.1 + hdf_compression_options: Optional[CompressionOption] = None data_path: Optional[str] = field(default_factory=lambda: gettempdir()) output_filename: Optional[str] = field(default_factory=lambda: f"{get_date_string()}_kwave_input.h5") input_filename: Optional[str] = field(default_factory=lambda: f"{get_date_string()}_kwave_output.h5") @@ -134,9 +153,9 @@ def __post_init__(self): h5_literals = get_h5_literals() self.hdf_compression_options = h5_literals.HDF_COMPRESSION_OPTIONS # check value is an integer between 0 and 9 - assert ((isinstance(self.hdf_compression_options, int) and (0 <= self.hdf_compression_options <= 9)) or - (isinstance(self.hdf_compression_options, str) and ((self.hdf_compression_options.lower() == 'lzf') or - (self.hdf_compression_options.lower() == 'szip')))), \ + assert ((isinstance(self.hdf_compression_options.value, int) and (0 <= self.hdf_compression_options.value <= 9)) or + (isinstance(self.hdf_compression_options.value, str) and ((self.hdf_compression_options.value.lower() == 'lzf') or + (self.hdf_compression_options.value.lower() == 'szip')))), \ "Optional input ''hdf_compression_options'' is false: must an integer be between 0-9 or either 'lzf' or 'szip'" assert np.isscalar(self.multi_axial_PML_ratio) and self.multi_axial_PML_ratio >= 0, \ diff --git a/kwave/utils/io.py b/kwave/utils/io.py index 9dcc1db4..0a1dc1fe 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -13,6 +13,7 @@ from .conversion import cast_to_type from .data import get_date_string from .dotdictionary import dotdict +from kwave.options.simulation_options import CompressionOption def get_h5_literals(): @@ -47,12 +48,15 @@ def get_h5_literals(): 'HDF_FILE_MINOR_VERSION': '2', # compression level: set to be same as default h5py - 'HDF_COMPRESSION_OPTIONS': 4 + 'HDF_COMPRESSION_OPTIONS': CompressionOption.GZIP_4 }) return literals -def write_matrix(filename, matrix: np.ndarray, matrix_name: str, compression_options: Union[int, str] = None, auto_chunk: bool = True): +def write_matrix(filename, matrix: np.ndarray, + matrix_name: str, + compression_options: Optional[CompressionOption] = None, + auto_chunk: bool = True): # get literals h5_literals = get_h5_literals() @@ -99,7 +103,7 @@ def write_matrix(filename, matrix: np.ndarray, matrix_name: str, compression_opt else: # set no compression - compression_options = 0 + compression_options = CompressionOption.GZIP_0 # set chunk size to grid size if matrix.size == 1: @@ -186,7 +190,7 @@ def write_matrix(filename, matrix: np.ndarray, matrix_name: str, compression_opt 'chunks': auto_chunk if auto_chunk is True else tuple(chunk_size) } - if compression_options != 0: + if compression_options != CompressionOption.GZIP_0: # use compression opts['compression'] = compression_options From 9db2fe931a65cb2469bb21303895c51e8c87bc93 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 5 Mar 2024 17:16:32 +0100 Subject: [PATCH 10/21] switch location --- kwave/options/simulation_options.py | 34 ++++++++++++++--------------- kwave/utils/io.py | 26 ++++++++++++++++++---- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/kwave/options/simulation_options.py b/kwave/options/simulation_options.py index 5856b6d5..63970ea4 100644 --- a/kwave/options/simulation_options.py +++ b/kwave/options/simulation_options.py @@ -12,7 +12,7 @@ # Found here: https://adamj.eu/tech/2021/05/13/python-type-hints-how-to-fix-circular-imports/ from kwave.kgrid import kWaveGrid from kwave.utils.data import get_date_string -from kwave.utils.io import get_h5_literals +from kwave.utils.io import get_h5_literals, CompressionOption from kwave.utils.pml import get_optimal_pml_size @@ -41,22 +41,22 @@ def is_axisymmetric(self): return self == SimulationType.AXISYMMETRIC -class CompressionOption(Enum): - """ - Enum for the compression options - """ - GZIP_0 = 0 - GZIP_1 = 1 - GZIP_2 = 2 - GZIP_3 = 3 - GZIP_4 = 4 - GZIP_5 = 5 - GZIP_6 = 6 - GZIP_7 = 7 - GZIP_8 = 8 - GZIP_9 = 9 - LZF = 'lzf' - SZIP = 'szip' +# class CompressionOption(Enum): +# """ +# Enum for the compression options +# """ +# GZIP_0 = 0 +# GZIP_1 = 1 +# GZIP_2 = 2 +# GZIP_3 = 3 +# GZIP_4 = 4 +# GZIP_5 = 5 +# GZIP_6 = 6 +# GZIP_7 = 7 +# GZIP_8 = 8 +# GZIP_9 = 9 +# LZF = 'lzf' +# SZIP = 'szip' diff --git a/kwave/utils/io.py b/kwave/utils/io.py index 0a1dc1fe..b06dd078 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -10,10 +10,28 @@ import numpy as np import kwave -from .conversion import cast_to_type -from .data import get_date_string -from .dotdictionary import dotdict -from kwave.options.simulation_options import CompressionOption +from kwave.conversion import cast_to_type +from kwave.data import get_date_string +from kwave.dotdictionary import dotdict + +# from kwave.options.simulation_options import CompressionOption + +class CompressionOption(Enum): + """ + Enum for the compression options + """ + GZIP_0 = 0 + GZIP_1 = 1 + GZIP_2 = 2 + GZIP_3 = 3 + GZIP_4 = 4 + GZIP_5 = 5 + GZIP_6 = 6 + GZIP_7 = 7 + GZIP_8 = 8 + GZIP_9 = 9 + LZF = 'lzf' + SZIP = 'szip' def get_h5_literals(): From ca790bb48d378a0762127c7727169fbe6de2b277 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 5 Mar 2024 17:18:32 +0100 Subject: [PATCH 11/21] revert to previous imports --- kwave/utils/io.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kwave/utils/io.py b/kwave/utils/io.py index b06dd078..5a4a2387 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -10,9 +10,9 @@ import numpy as np import kwave -from kwave.conversion import cast_to_type -from kwave.data import get_date_string -from kwave.dotdictionary import dotdict +from .conversion import cast_to_type +from .data import get_date_string +from .dotdictionary import dotdict # from kwave.options.simulation_options import CompressionOption From 78580faa1b8898288fcf0c3340b246a6468444d8 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 5 Mar 2024 17:22:54 +0100 Subject: [PATCH 12/21] fix import --- kwave/utils/io.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kwave/utils/io.py b/kwave/utils/io.py index 5a4a2387..46318357 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -2,6 +2,7 @@ import os import platform import socket +from enum import Enum from datetime import datetime from typing import Optional, Union From 40a28c04b3d855ef48bb81e243ff97c1b0a5c788 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 5 Mar 2024 20:18:25 +0100 Subject: [PATCH 13/21] test value getter --- kwave/utils/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kwave/utils/io.py b/kwave/utils/io.py index 46318357..13b51fd1 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -211,7 +211,7 @@ def write_matrix(filename, matrix: np.ndarray, if compression_options != CompressionOption.GZIP_0: # use compression - opts['compression'] = compression_options + opts['compression'] = compression_options.value # write the matrix into the file with h5py.File(filename, "a") as f: From 6dc2159344d75b6a11f5dfdc577a1ef0bfa8a414 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 5 Mar 2024 20:26:53 +0100 Subject: [PATCH 14/21] remove bad setter --- kwave/utils/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kwave/utils/io.py b/kwave/utils/io.py index 13b51fd1..46318357 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -211,7 +211,7 @@ def write_matrix(filename, matrix: np.ndarray, if compression_options != CompressionOption.GZIP_0: # use compression - opts['compression'] = compression_options.value + opts['compression'] = compression_options # write the matrix into the file with h5py.File(filename, "a") as f: From 5597b6e8e89d4d10fca37fe0d9f4fb89c45d315d Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 5 Mar 2024 20:41:12 +0100 Subject: [PATCH 15/21] using values --- kwave/utils/io.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kwave/utils/io.py b/kwave/utils/io.py index 46318357..e609f53a 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -67,7 +67,7 @@ def get_h5_literals(): 'HDF_FILE_MINOR_VERSION': '2', # compression level: set to be same as default h5py - 'HDF_COMPRESSION_OPTIONS': CompressionOption.GZIP_4 + 'HDF_COMPRESSION_OPTIONS': CompressionOption.GZIP_4.value }) return literals @@ -122,7 +122,7 @@ def write_matrix(filename, matrix: np.ndarray, else: # set no compression - compression_options = CompressionOption.GZIP_0 + compression_options = CompressionOption.GZIP_0.value # set chunk size to grid size if matrix.size == 1: @@ -209,9 +209,9 @@ def write_matrix(filename, matrix: np.ndarray, 'chunks': auto_chunk if auto_chunk is True else tuple(chunk_size) } - if compression_options != CompressionOption.GZIP_0: + if compression_options != CompressionOption.GZIP_0.value: # use compression - opts['compression'] = compression_options + opts['compression'] = compression_options.value # write the matrix into the file with h5py.File(filename, "a") as f: From b36064f4a53619b1f33b4a62104329b66b79a4c4 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 5 Mar 2024 20:44:15 +0100 Subject: [PATCH 16/21] remove redundant check --- kwave/options/simulation_options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kwave/options/simulation_options.py b/kwave/options/simulation_options.py index 63970ea4..775d2ea3 100644 --- a/kwave/options/simulation_options.py +++ b/kwave/options/simulation_options.py @@ -153,9 +153,9 @@ def __post_init__(self): h5_literals = get_h5_literals() self.hdf_compression_options = h5_literals.HDF_COMPRESSION_OPTIONS # check value is an integer between 0 and 9 - assert ((isinstance(self.hdf_compression_options.value, int) and (0 <= self.hdf_compression_options.value <= 9)) or - (isinstance(self.hdf_compression_options.value, str) and ((self.hdf_compression_options.value.lower() == 'lzf') or - (self.hdf_compression_options.value.lower() == 'szip')))), \ + assert ((isinstance(self.hdf_compression_options, int) and (0 <= self.hdf_compression_options <= 9)) or + (isinstance(self.hdf_compression_options, str) and ((self.hdf_compression_options.lower() == 'lzf') or + (self.hdf_compression_options.lower() == 'szip')))), \ "Optional input ''hdf_compression_options'' is false: must an integer be between 0-9 or either 'lzf' or 'szip'" assert np.isscalar(self.multi_axial_PML_ratio) and self.multi_axial_PML_ratio >= 0, \ From f2f56ad137aa1a39c826795ba0cab027140afcec Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 5 Mar 2024 20:50:48 +0100 Subject: [PATCH 17/21] consistent handling of enum --- kwave/utils/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kwave/utils/io.py b/kwave/utils/io.py index e609f53a..2b38336e 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -211,7 +211,7 @@ def write_matrix(filename, matrix: np.ndarray, if compression_options != CompressionOption.GZIP_0.value: # use compression - opts['compression'] = compression_options.value + opts['compression'] = compression_options # write the matrix into the file with h5py.File(filename, "a") as f: From 57b7f86bbc1c62040511e886ef0aaca3ed581b1a Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 5 Mar 2024 22:10:02 +0100 Subject: [PATCH 18/21] remove unused imports --- kwave/options/simulation_options.py | 20 +------------------- kwave/utils/io.py | 2 +- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/kwave/options/simulation_options.py b/kwave/options/simulation_options.py index 775d2ea3..3b2dc6d3 100644 --- a/kwave/options/simulation_options.py +++ b/kwave/options/simulation_options.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field from enum import Enum from tempfile import gettempdir -from typing import List, Union, Optional, TYPE_CHECKING +from typing import List, Optional, TYPE_CHECKING import numpy as np @@ -41,24 +41,6 @@ def is_axisymmetric(self): return self == SimulationType.AXISYMMETRIC -# class CompressionOption(Enum): -# """ -# Enum for the compression options -# """ -# GZIP_0 = 0 -# GZIP_1 = 1 -# GZIP_2 = 2 -# GZIP_3 = 3 -# GZIP_4 = 4 -# GZIP_5 = 5 -# GZIP_6 = 6 -# GZIP_7 = 7 -# GZIP_8 = 8 -# GZIP_9 = 9 -# LZF = 'lzf' -# SZIP = 'szip' - - @dataclass class SimulationOptions(object): diff --git a/kwave/utils/io.py b/kwave/utils/io.py index 2b38336e..55417134 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -4,7 +4,7 @@ import socket from enum import Enum from datetime import datetime -from typing import Optional, Union +from typing import Optional import cv2 import h5py From 6a5c183546307184312e5f73f1ce352ed11da5d1 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Tue, 5 Mar 2024 22:47:15 +0100 Subject: [PATCH 19/21] change for default values --- kwave/options/simulation_options.py | 2 +- kwave/utils/io.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/kwave/options/simulation_options.py b/kwave/options/simulation_options.py index 3b2dc6d3..c2c46f19 100644 --- a/kwave/options/simulation_options.py +++ b/kwave/options/simulation_options.py @@ -107,7 +107,7 @@ class SimulationOptions(object): pml_search_range: List[int] = field(default_factory=lambda: [10, 40]) radial_symmetry: str = 'WSWA-FFT' multi_axial_PML_ratio: float = 0.1 - hdf_compression_options: Optional[CompressionOption] = None + hdf_compression_options: Optional[CompressionOption] = CompressionOption.GZIP_4.value data_path: Optional[str] = field(default_factory=lambda: gettempdir()) output_filename: Optional[str] = field(default_factory=lambda: f"{get_date_string()}_kwave_input.h5") input_filename: Optional[str] = field(default_factory=lambda: f"{get_date_string()}_kwave_output.h5") diff --git a/kwave/utils/io.py b/kwave/utils/io.py index 55417134..9096fe43 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -74,16 +74,13 @@ def get_h5_literals(): def write_matrix(filename, matrix: np.ndarray, matrix_name: str, - compression_options: Optional[CompressionOption] = None, + compression_options: Optional[CompressionOption] = CompressionOption.GZIP_4.value, auto_chunk: bool = True): # get literals h5_literals = get_h5_literals() assert isinstance(auto_chunk, bool), "auto_chunk must be a boolean." - if compression_options is None: - compression_options = h5_literals.HDF_COMPRESSION_OPTIONS - # dims = num_dim(matrix) dims = len(matrix.shape) From b5940ce981945634f25fc8c64325c3fcfc9d30a9 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Wed, 6 Mar 2024 09:10:47 +0100 Subject: [PATCH 20/21] change default value --- kwave/utils/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kwave/utils/io.py b/kwave/utils/io.py index 9096fe43..b75d520e 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -74,7 +74,7 @@ def get_h5_literals(): def write_matrix(filename, matrix: np.ndarray, matrix_name: str, - compression_options: Optional[CompressionOption] = CompressionOption.GZIP_4.value, + compression_options: Optional[CompressionOption] = h5_literals.HDF_COMPRESSION_OPTIONS, auto_chunk: bool = True): # get literals h5_literals = get_h5_literals() From 4b1a5632d878580ebe0b46061e2375fd7d6403c7 Mon Sep 17 00:00:00 2001 From: David Sinden Date: Wed, 6 Mar 2024 09:16:53 +0100 Subject: [PATCH 21/21] another default approach --- kwave/utils/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kwave/utils/io.py b/kwave/utils/io.py index b75d520e..47f11e4d 100644 --- a/kwave/utils/io.py +++ b/kwave/utils/io.py @@ -74,7 +74,7 @@ def get_h5_literals(): def write_matrix(filename, matrix: np.ndarray, matrix_name: str, - compression_options: Optional[CompressionOption] = h5_literals.HDF_COMPRESSION_OPTIONS, + compression_options: Optional[CompressionOption] = get_h5_literals().HDF_COMPRESSION_OPTIONS, auto_chunk: bool = True): # get literals h5_literals = get_h5_literals()