From a5d6455abb9db768d20d3f1bff39322e60339692 Mon Sep 17 00:00:00 2001 From: Kristof Herrmann Date: Sun, 5 May 2024 17:44:01 +0200 Subject: [PATCH 1/4] fix: total filtered download --- deepset_cloud_sdk/_service/files_service.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/deepset_cloud_sdk/_service/files_service.py b/deepset_cloud_sdk/_service/files_service.py index 9b22d145..f05819d4 100644 --- a/deepset_cloud_sdk/_service/files_service.py +++ b/deepset_cloud_sdk/_service/files_service.py @@ -490,7 +490,15 @@ async def download( pbar: Optional[tqdm] = None if show_progress: - total = (await self._files.list_paginated(workspace_name, limit=1)).total + total = ( + await self._files.list_paginated( + workspace_name, + name=name, + content=content, + odata_filter=odata_filter, + limit=1, + ) + ).total pbar = tqdm(total=total, desc="Download Progress") after_value = None From ab8775fbe82ae84d81dbcb5c87c657ffb138ba20 Mon Sep 17 00:00:00 2001 From: Kristof Herrmann Date: Mon, 6 May 2024 09:26:58 +0200 Subject: [PATCH 2/4] tests: check for total count --- tests/unit/service/test_files_service.py | 55 ++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/unit/service/test_files_service.py b/tests/unit/service/test_files_service.py index 42d42231..afaae496 100644 --- a/tests/unit/service/test_files_service.py +++ b/tests/unit/service/test_files_service.py @@ -681,6 +681,61 @@ async def test_download_files_with_filter(self, file_service: FilesService, monk after_value=None, ) + async def test_download_files_with_filter_and_progress_bar( + self, file_service: FilesService, monkeypatch: MonkeyPatch + ) -> None: + mocked_list_paginated = AsyncMock( + side_effect=[ + FileList( + total=1, + data=[ + File( + file_id=UUID("cd16435f-f6eb-423f-bf6f-994dc8a36a10"), + url="/api/v1/workspaces/search tests/files/cd16435f-f6eb-423f-bf6f-994dc8a36a10", + name="silly_things_2.txt", + size=611, + created_at=datetime.datetime.fromisoformat("2022-06-21T16:40:00.634653+00:00"), + meta={}, + ) + ], + has_more=False, + ), + ] + ) + + monkeypatch.setattr(file_service._files, "list_paginated", mocked_list_paginated) + + mocked_download = AsyncMock(return_value=None) + monkeypatch.setattr(file_service._files, "download", mocked_download) + + await file_service.download( + workspace_name="test_workspace", + show_progress=True, # This requires a previous cal that checks the total number of files + odata_filter="category eq 'news'", + name="asdf", + content="bsdf", + batch_size=54, + ) + + mocked_list_paginated.mock_calls == [ + call( + workspace_name="test_workspace", + name="asdf", + content="bsdf", + odata_filter="category eq 'news'", + limit=54, + ), + call( + workspace_name="test_workspace", + name="asdf", + content="bsdf", + odata_filter="category eq 'news'", + limit=54, + after_file_id=None, + after_value=None, + ), + ] + async def test_download_all_files_with_file_not_found( self, file_service: FilesService, monkeypatch: MonkeyPatch ) -> None: From 12571e61fed32fc28b460e8d7837595a80835a96 Mon Sep 17 00:00:00 2001 From: Kristof Herrmann Date: Mon, 6 May 2024 09:27:58 +0200 Subject: [PATCH 3/4] fix: test --- tests/unit/service/test_files_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/service/test_files_service.py b/tests/unit/service/test_files_service.py index afaae496..5490faf3 100644 --- a/tests/unit/service/test_files_service.py +++ b/tests/unit/service/test_files_service.py @@ -685,7 +685,7 @@ async def test_download_files_with_filter_and_progress_bar( self, file_service: FilesService, monkeypatch: MonkeyPatch ) -> None: mocked_list_paginated = AsyncMock( - side_effect=[ + return_value=[ FileList( total=1, data=[ From f1cce3f8a8db02186fe2eb87c21def571d7a695b Mon Sep 17 00:00:00 2001 From: Kristof Herrmann Date: Mon, 6 May 2024 09:38:50 +0200 Subject: [PATCH 4/4] fix --- tests/unit/service/test_files_service.py | 30 +++++++++++------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/tests/unit/service/test_files_service.py b/tests/unit/service/test_files_service.py index 5490faf3..cfe4a23f 100644 --- a/tests/unit/service/test_files_service.py +++ b/tests/unit/service/test_files_service.py @@ -685,22 +685,20 @@ async def test_download_files_with_filter_and_progress_bar( self, file_service: FilesService, monkeypatch: MonkeyPatch ) -> None: mocked_list_paginated = AsyncMock( - return_value=[ - FileList( - total=1, - data=[ - File( - file_id=UUID("cd16435f-f6eb-423f-bf6f-994dc8a36a10"), - url="/api/v1/workspaces/search tests/files/cd16435f-f6eb-423f-bf6f-994dc8a36a10", - name="silly_things_2.txt", - size=611, - created_at=datetime.datetime.fromisoformat("2022-06-21T16:40:00.634653+00:00"), - meta={}, - ) - ], - has_more=False, - ), - ] + return_value=FileList( + total=1, + data=[ + File( + file_id=UUID("cd16435f-f6eb-423f-bf6f-994dc8a36a10"), + url="/api/v1/workspaces/search tests/files/cd16435f-f6eb-423f-bf6f-994dc8a36a10", + name="silly_things_2.txt", + size=611, + created_at=datetime.datetime.fromisoformat("2022-06-21T16:40:00.634653+00:00"), + meta={}, + ) + ], + has_more=False, + ), ) monkeypatch.setattr(file_service._files, "list_paginated", mocked_list_paginated)