Enable run cross-validation without training workflow and examples

examples/hello-world/hello-numpy-cross-val/README.md

@@ -27,3 +27,28 @@ $ ls /tmp/nvflare/simulate_job/
 app_server  app_site-1  app_site-2  log.txt
 
 ```
+
+# Run cross site validation using the previous trained results
+
+## Introduction
+
+The "hello-numpy-cross-val-only" and "hello-numpy-cross-val-only-list-models" jobs show how to run the NVFlare cross-site validation without the training workflow, making use of the previous run results. The first one uses the default single server model. The second enables a list of server models. You can provide / use your own previous trained models for the cross-validation.
+
+### Generate the previous run best global model and local best model
+
+Run the following command to generate the pre-trained models:
+
+```
+python pre_train_models.py 
+```
+
+### How to run the Job
+
+Define two OS system variable "SERVER_MODEL_DIR" and "CLIENT_MODEL_DIR" to point to the absolute path of the server best model and local best model location respectively. Then use the NVFlare admin command "submit_job" to submit and run the cross-validation job.
+
+For example, define the system variable "SERVER_MODEL_DIR" like this:
+
+```
+export SERVER_MODEL_DIR="/path/to/model/location/at/server-side"
+```
+

examples/hello-world/hello-numpy-cross-val/jobs/hello-numpy-cross-val-only-list-models/app/config/config_fed_client.json

@@ -0,0 +1,29 @@
+{
+  "format_version": 2,
+  "model_dir": "{$CLIENT_MODEL_DIR}",
+  "executors": [
+    {
+      "tasks": [
+        "train",
+        "submit_model"
+      ],
+      "executor": {
+        "path": "nvflare.app_common.np.np_trainer.NPTrainer",
+        "args": {
+          "model_dir": "{model_dir}"
+        }
+      }
+    },
+    {
+      "tasks": [
+        "validate"
+      ],
+      "executor": {
+        "path": "nvflare.app_common.np.np_validator.NPValidator"
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": [],
+  "components": []
+}

examples/hello-world/hello-numpy-cross-val/jobs/hello-numpy-cross-val-only-list-models/app/config/config_fed_server.json

@@ -0,0 +1,39 @@
+{
+  "format_version": 2,
+  "model_dir": "{$SERVER_MODEL_DIR}",
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "model_locator",
+      "path": "nvflare.app_common.np.np_model_locator.NPModelLocator",
+      "args": {
+        "model_dir": "{model_dir}",
+        "model_names": {
+          "server_model_1": "server_1.npy",
+          "server_model_2": "server_2.npy"
+        }
+      }
+    },
+    {
+      "id": "json_generator",
+      "path": "nvflare.app_common.widgets.validation_json_generator.ValidationJsonGenerator",
+      "args": {}
+    }
+  ],
+  "workflows": [
+    {
+      "id": "cross_site_model_eval",
+      "path": "nvflare.app_common.workflows.cross_site_model_eval.CrossSiteModelEval",
+      "args": {
+        "model_locator_id": "model_locator",
+        "submit_model_timeout": 600,
+        "validation_timeout": 6000,
+        "cleanup_models": false
+      }
+    }
+  ]
+}

examples/hello-world/hello-numpy-cross-val/jobs/hello-numpy-cross-val-only-list-models/meta.json

@@ -0,0 +1,10 @@
+{
+  "name": "hello-numpy-cross-val",
+  "resource_spec": {},
+  "min_clients" : 2,
+  "deploy_map": {
+    "app": [
+      "@ALL"
+    ]
+  }
+}

examples/hello-world/hello-numpy-cross-val/jobs/hello-numpy-cross-val-only-list-models/pre_train_models.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import numpy as np
+
+from nvflare.app_common.abstract.model import ModelLearnableKey, make_model_learnable
+from nvflare.app_common.np.constants import NPConstants
+
+SERVER_MODEL_DIR = "models/server"
+CLIENT_MODEL_DIR = "models/client"
+
+if __name__ == "__main__":
+    """
+    This is the tool to generate the pre-trained models for demonstrating the cross-validation without training.
+    """
+
+    model_data = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32)
+    model_learnable = make_model_learnable(weights={NPConstants.NUMPY_KEY: model_data}, meta_props={})
+
+    working_dir = os.getcwd()
+    model_dir = os.path.join(working_dir, SERVER_MODEL_DIR)
+    if not os.path.exists(model_dir):
+        os.makedirs(model_dir)
+    model_path = os.path.join(model_dir, "server_1.npy")
+    np.save(model_path, model_learnable[ModelLearnableKey.WEIGHTS][NPConstants.NUMPY_KEY])
+    model_path = os.path.join(model_dir, "server_2.npy")
+    np.save(model_path, model_learnable[ModelLearnableKey.WEIGHTS][NPConstants.NUMPY_KEY])
+
+    model_dir = os.path.join(working_dir, CLIENT_MODEL_DIR)
+    if not os.path.exists(model_dir):
+        os.makedirs(model_dir)
+    model_save_path = os.path.join(model_dir, "best_numpy.npy")
+    np.save(model_save_path, model_data)

examples/hello-world/hello-numpy-cross-val/jobs/hello-numpy-cross-val-only/app/config/config_fed_client.json

@@ -0,0 +1,29 @@
+{
+  "format_version": 2,
+  "model_dir": "{$CLIENT_MODEL_DIR}",
+  "executors": [
+    {
+      "tasks": [
+        "train",
+        "submit_model"
+      ],
+      "executor": {
+        "path": "nvflare.app_common.np.np_trainer.NPTrainer",
+        "args": {
+          "model_dir": "{model_dir}"
+        }
+      }
+    },
+    {
+      "tasks": [
+        "validate"
+      ],
+      "executor": {
+        "path": "nvflare.app_common.np.np_validator.NPValidator"
+      }
+    }
+  ],
+  "task_result_filters": [],
+  "task_data_filters": [],
+  "components": []
+}

examples/hello-world/hello-numpy-cross-val/jobs/hello-numpy-cross-val-only/app/config/config_fed_server.json

@@ -0,0 +1,35 @@
+{
+  "format_version": 2,
+  "model_dir": "{$SERVER_MODEL_DIR}",
+  "server": {
+    "heart_beat_timeout": 600
+  },
+  "task_data_filters": [],
+  "task_result_filters": [],
+  "components": [
+    {
+      "id": "model_locator",
+      "path": "nvflare.app_common.np.np_model_locator.NPModelLocator",
+      "args": {
+        "model_dir": "{model_dir}"
+      }
+    },
+    {
+      "id": "json_generator",
+      "path": "nvflare.app_common.widgets.validation_json_generator.ValidationJsonGenerator",
+      "args": {}
+    }
+  ],
+  "workflows": [
+    {
+      "id": "cross_site_model_eval",
+      "path": "nvflare.app_common.workflows.cross_site_model_eval.CrossSiteModelEval",
+      "args": {
+        "model_locator_id": "model_locator",
+        "submit_model_timeout": 600,
+        "validation_timeout": 6000,
+        "cleanup_models": false
+      }
+    }
+  ]
+}

examples/hello-world/hello-numpy-cross-val/jobs/hello-numpy-cross-val-only/meta.json

@@ -0,0 +1,10 @@
+{
+  "name": "hello-numpy-cross-val",
+  "resource_spec": {},
+  "min_clients" : 2,
+  "deploy_map": {
+    "app": [
+      "@ALL"
+    ]
+  }
+}

examples/hello-world/hello-numpy-cross-val/jobs/hello-numpy-cross-val-only/pre_train_models.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import numpy as np
+
+from nvflare.app_common.abstract.model import ModelLearnableKey, make_model_learnable
+from nvflare.app_common.np.constants import NPConstants
+
+SERVER_MODEL_DIR = "models/server"
+CLIENT_MODEL_DIR = "models/client"
+
+if __name__ == "__main__":
+    """
+    This is the tool to generate the pre-trained models for demonstrating the cross-validation without training.
+    """
+
+    model_data = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=np.float32)
+    model_learnable = make_model_learnable(weights={NPConstants.NUMPY_KEY: model_data}, meta_props={})
+
+    working_dir = os.getcwd()
+    model_dir = os.path.join(working_dir, SERVER_MODEL_DIR)
+    if not os.path.exists(model_dir):
+        os.makedirs(model_dir)
+    model_path = os.path.join(model_dir, "server.npy")
+    np.save(model_path, model_learnable[ModelLearnableKey.WEIGHTS][NPConstants.NUMPY_KEY])
+
+    model_dir = os.path.join(working_dir, CLIENT_MODEL_DIR)
+    if not os.path.exists(model_dir):
+        os.makedirs(model_dir)
+    model_save_path = os.path.join(model_dir, "best_numpy.npy")
+    np.save(model_save_path, model_data)

nvflare/app_common/model_locator/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nvflare/app_common/model_locator/list_model_locator.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from nvflare.apis.dxo import DXO
+from nvflare.apis.event_type import EventType
+from nvflare.apis.fl_context import FLContext
+from nvflare.app_common.abstract.model import model_learnable_to_dxo
+from nvflare.app_common.abstract.model_locator import ModelLocator
+from nvflare.app_common.abstract.model_persistor import ModelPersistor
+
+
+class ListModelLocator(ModelLocator):
+    def __init__(self, persistor_id: str, model_list={}):
+        """The ModelLocator's job is to find and locate the models inventory saved during training.
+
+        Args:
+            persistor_id (str): ModelPersistor component ID
+            model_list: a list of model_names and locations
+        """
+        super().__init__()
+
+        self.persistor_id = persistor_id
+
+        self.model_persistor = None
+        self.model_list = model_list
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        if event_type == EventType.START_RUN:
+            self._initialize(fl_ctx)
+
+    def _initialize(self, fl_ctx: FLContext):
+        engine = fl_ctx.get_engine()
+        self.model_persistor: ModelPersistor = engine.get_component(self.persistor_id)
+        if self.model_persistor is None or not isinstance(self.model_persistor, ModelPersistor):
+            raise ValueError(
+                f"persistor_id component must be PTFileModelPersistor. " f"But got: {type(self.model_persistor)}"
+            )
+
+    def get_model_names(self, fl_ctx: FLContext) -> List[str]:
+        """Returns the list of model names that should be included from server in cross site validation.add().
+
+        Args:
+            fl_ctx (FLContext): FL Context object.
+
+        Returns:
+            List[str]: List of model names.
+        """
+        return list(self.model_list.keys())
+
+    def locate_model(self, model_name, fl_ctx: FLContext) -> DXO:
+        """Call to locate and load the model weights of model_name.
+
+        Args:
+            model_name: name of the model
+            fl_ctx: FLContext
+
+        Returns: model_weight DXO
+
+        """
+        if model_name not in list(self.model_list.keys()):
+            raise ValueError(f"model inventory does not contain: {model_name}")
+
+        location = self.model_list[model_name]
+        model_learnable = self.model_persistor.get_model_from_location(location, fl_ctx)
+        dxo = model_learnable_to_dxo(model_learnable)
+
+        return dxo