From fa0492683a41b2c58f59c86dc5ddd19022acbae9 Mon Sep 17 00:00:00 2001
From: Ben Galewsky <ben@peartreestudio.net>
Date: Wed, 13 Nov 2024 12:23:39 -0600
Subject: [PATCH] Allow users to view deleted datasets

---
 docs/command_line.rst          |  4 +++-
 servicex/app/datasets.py       | 11 ++++++++++-
 servicex/models.py             |  1 +
 servicex/servicex_adapter.py   |  5 ++++-
 servicex/servicex_client.py    |  4 ++--
 tests/test_servicex_adapter.py |  4 +++-
 6 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/docs/command_line.rst b/docs/command_line.rst
index 819144bb..51d6bf70 100644
--- a/docs/command_line.rst
+++ b/docs/command_line.rst
@@ -85,7 +85,9 @@ list
 ^^^^
 List all of the datasets cached on the server. Accepts a command line argument
 of ``--did-finder`` to filter the list of datasets by a specific DID finder such
-as ``rucio`` or ``user``.
+as ``rucio`` or ``user``. By default this will filter out deleted datasets. They
+are still in the cache, but marked as stale. To see deleted datasets, use the
+``--show-deleted`` flag.
 
 get
 ^^^
diff --git a/servicex/app/datasets.py b/servicex/app/datasets.py
index c897f892..0ee9ee29 100644
--- a/servicex/app/datasets.py
+++ b/servicex/app/datasets.py
@@ -49,6 +49,11 @@ def list(
             help="Filter datasets by DID finder. Some useful values are 'rucio' or 'user'",
             show_default=False,
         ),
+        show_deleted: Optional[bool] = typer.Option(
+            False,
+            help="Show deleted datasets",
+            show_default=True,
+        ),
 ):
     """
     List the datasets.
@@ -61,7 +66,10 @@ def list(
     table.add_column("Size")
     table.add_column("Status")
     table.add_column("Created")
-    datasets = asyncio.run(sx.get_datasets(did_finder=did_finder))
+    if show_deleted:
+        table.add_column("Deleted")
+
+    datasets = asyncio.run(sx.get_datasets(did_finder=did_finder, show_deleted=show_deleted))
     for d in datasets:
         # Format the CachedDataset object into a table row
         # The last_updated field is what we should be displaying, but that is
@@ -75,6 +83,7 @@ def list(
             "{:,}MB".format(round(d.size / 1e6)),
             d.lookup_status,
             d.last_used.strftime('%Y-%m-%dT%H:%M:%S'),
+            "Yes" if d.is_stale else ""
         )
     rich.print(table)
 
diff --git a/servicex/models.py b/servicex/models.py
index 717c674c..8aa7b87d 100644
--- a/servicex/models.py
+++ b/servicex/models.py
@@ -196,4 +196,5 @@ class CachedDataset(BaseModel):
     last_used: datetime
     last_updated: datetime
     lookup_status: str
+    is_stale: bool
     files: Optional[List[DatasetFile]] = []
diff --git a/servicex/servicex_adapter.py b/servicex/servicex_adapter.py
index 002a825a..9613379f 100644
--- a/servicex/servicex_adapter.py
+++ b/servicex/servicex_adapter.py
@@ -109,11 +109,14 @@ def get_code_generators(self):
                     f"Not authorized to access serviceX at {self.url}")
             return r.json()
 
-    async def get_datasets(self, did_finder=None) -> List[CachedDataset]:
+    async def get_datasets(self, did_finder=None, show_deleted=False) -> List[CachedDataset]:
         headers = await self._get_authorization()
 
         with httpx.Client() as client:
             params = {"did-finder": did_finder} if did_finder else {}
+            if show_deleted:
+                params['show-deleted'] = True
+
             r = client.get(headers=headers,
                            url=f"{self.url}/servicex/datasets",
                            params=params)
diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py
index 842c0b23..b0cbbf4a 100644
--- a/servicex/servicex_client.py
+++ b/servicex/servicex_client.py
@@ -280,12 +280,12 @@ async def get_transform_status_async(self, transform_id) -> TransformStatus:
 
     get_transform_status = make_sync(get_transform_status_async)
 
-    def get_datasets(self, did_finder=None):
+    def get_datasets(self, did_finder=None, show_deleted=False):
         r"""
         Retrieve all datasets you have run on the server
         :return: List of Query objects
         """
-        return self.servicex.get_datasets(did_finder)
+        return self.servicex.get_datasets(did_finder, show_deleted)
 
     def get_dataset(self, dataset_id):
         r"""
diff --git a/tests/test_servicex_adapter.py b/tests/test_servicex_adapter.py
index bb93c783..7f631a07 100644
--- a/tests/test_servicex_adapter.py
+++ b/tests/test_servicex_adapter.py
@@ -143,7 +143,8 @@ async def test_get_datasets(get, servicex):
                 "last_used": "2022-01-01T00:00:00.000000Z",
                 "last_updated": "2022-01-01T00:00:00.000000Z",
                 "lookup_status": "looking",
-                "did_finder": "rucio"
+                "did_finder": "rucio",
+                "is_stale": False
             }
 
         ]
@@ -175,6 +176,7 @@ async def test_get_dataset(get, servicex):
         "last_used": "2024-11-12T01:59:19.161655Z",
         "last_updated": "1969-12-31T18:00:00.000000Z",
         "lookup_status": "complete",
+        "is_stale": False,
         "files": [
             {
                 "id": 12,