Allow users to view deleted datasets

docs/command_line.rst

@@ -85,7 +85,9 @@ list
 ^^^^
 List all of the datasets cached on the server. Accepts a command line argument
 of ``--did-finder`` to filter the list of datasets by a specific DID finder such
-as ``rucio`` or ``user``.
+as ``rucio`` or ``user``. By default this will filter out deleted datasets. They
+are still in the cache, but marked as stale. To see deleted datasets, use the
+``--show-deleted`` flag.
 
 get
 ^^^

servicex/app/datasets.py

@@ -49,6 +49,11 @@ def list(
             help="Filter datasets by DID finder. Some useful values are 'rucio' or 'user'",
             show_default=False,
         ),
+        show_deleted: Optional[bool] = typer.Option(
+            False,
+            help="Show deleted datasets",
+            show_default=True,
+        ),
 ):
     """
     List the datasets.
@@ -61,7 +66,10 @@ def list(
     table.add_column("Size")
     table.add_column("Status")
     table.add_column("Created")
-    datasets = asyncio.run(sx.get_datasets(did_finder=did_finder))
+    if show_deleted:
+        table.add_column("Deleted")
+
+    datasets = asyncio.run(sx.get_datasets(did_finder=did_finder, show_deleted=show_deleted))
     for d in datasets:
         # Format the CachedDataset object into a table row
         # The last_updated field is what we should be displaying, but that is
@@ -75,6 +83,7 @@ def list(
             "{:,}MB".format(round(d.size / 1e6)),
             d.lookup_status,
             d.last_used.strftime('%Y-%m-%dT%H:%M:%S'),
+            "Yes" if d.is_stale else ""
         )
     rich.print(table)
 

servicex/models.py

@@ -196,4 +196,5 @@ class CachedDataset(BaseModel):
     last_used: datetime
     last_updated: datetime
     lookup_status: str
+    is_stale: bool
     files: Optional[List[DatasetFile]] = []

servicex/servicex_adapter.py

@@ -109,11 +109,14 @@ def get_code_generators(self):
                     f"Not authorized to access serviceX at {self.url}")
             return r.json()
 
-    async def get_datasets(self, did_finder=None) -> List[CachedDataset]:
+    async def get_datasets(self, did_finder=None, show_deleted=False) -> List[CachedDataset]:
         headers = await self._get_authorization()
 
         with httpx.Client() as client:
             params = {"did-finder": did_finder} if did_finder else {}
+            if show_deleted:
+                params['show-deleted'] = True
+
             r = client.get(headers=headers,
                            url=f"{self.url}/servicex/datasets",
                            params=params)

servicex/servicex_client.py

@@ -280,12 +280,12 @@ async def get_transform_status_async(self, transform_id) -> TransformStatus:
 
     get_transform_status = make_sync(get_transform_status_async)
 
-    def get_datasets(self, did_finder=None):
+    def get_datasets(self, did_finder=None, show_deleted=False):
         r"""
         Retrieve all datasets you have run on the server
         :return: List of Query objects
         """
-        return self.servicex.get_datasets(did_finder)
+        return self.servicex.get_datasets(did_finder, show_deleted)
 
     def get_dataset(self, dataset_id):
         r"""

tests/test_servicex_adapter.py

@@ -143,7 +143,8 @@ async def test_get_datasets(get, servicex):
                 "last_used": "2022-01-01T00:00:00.000000Z",
                 "last_updated": "2022-01-01T00:00:00.000000Z",
                 "lookup_status": "looking",
-                "did_finder": "rucio"
+                "did_finder": "rucio",
+                "is_stale": False
             }
 
         ]
@@ -175,6 +176,7 @@ async def test_get_dataset(get, servicex):
         "last_used": "2024-11-12T01:59:19.161655Z",
         "last_updated": "1969-12-31T18:00:00.000000Z",
         "lookup_status": "complete",
+        "is_stale": False,
         "files": [
             {
                 "id": 12,