From 9785297bb529b3ebad5915d2c9daedd4203ab7bd Mon Sep 17 00:00:00 2001 From: Sean Yang Date: Thu, 22 Aug 2024 13:44:51 -0700 Subject: [PATCH] ScriptRunner framework option in examples (#2827) * use framework option in examples * rename files --- .../job_api/sklearn/kmeans_script_runner_higgs.py | 5 ++--- .../advanced/job_api/tf/tf_fl_script_runner_cifar10.py | 6 ++++-- .../sklearn/kmeans_script_runner_higgs.py | 5 ++--- .../getting_started/tf/nvflare_tf_getting_started.ipynb | 6 ++---- .../getting_started/tf/tf_fl_script_runner_cifar10.py | 6 ++++-- examples/hello-world/hello-cyclic/README.md | 2 +- ...pt_runner_hello-cyclic.py => cyclic_script_runner.py} | 9 ++++----- .../fedavg_script_runner_hello-numpy.py | 5 ++--- .../hello-fedavg-numpy_getting_started.ipynb | 6 ++---- examples/hello-world/hello-pt/README.md | 2 +- ...ipt_runner_hello-pt.py => fedavg_script_runner_pt.py} | 0 examples/hello-world/hello-tf/README.md | 2 +- ...ipt_runner_hello-tf.py => fedavg_script_runner_tf.py} | 6 ++++-- 13 files changed, 29 insertions(+), 31 deletions(-) rename examples/hello-world/hello-cyclic/{cyclic_script_runner_hello-cyclic.py => cyclic_script_runner.py} (85%) rename examples/hello-world/hello-pt/{fedavg_script_runner_hello-pt.py => fedavg_script_runner_pt.py} (100%) rename examples/hello-world/hello-tf/{fedavg_script_runner_hello-tf.py => fedavg_script_runner_tf.py} (83%) diff --git a/examples/advanced/job_api/sklearn/kmeans_script_runner_higgs.py b/examples/advanced/job_api/sklearn/kmeans_script_runner_higgs.py index 592b934d8f..d1aaaf6b1d 100644 --- a/examples/advanced/job_api/sklearn/kmeans_script_runner_higgs.py +++ b/examples/advanced/job_api/sklearn/kmeans_script_runner_higgs.py @@ -24,8 +24,7 @@ from nvflare.app_common.widgets.intime_model_selector import IntimeModelSelector from nvflare.app_common.workflows.scatter_and_gather import ScatterAndGather from nvflare.app_opt.sklearn.joblib_model_param_persistor import JoblibModelParamPersistor -from nvflare.client.config import ExchangeFormat -from nvflare.job_config.script_runner import ScriptRunner +from nvflare.job_config.script_runner import FrameworkType, ScriptRunner preprocess = True # if False, assume data is already preprocessed and split @@ -146,7 +145,7 @@ def split_higgs(input_data_path, input_header_path, output_dir, site_num, sample executor = ScriptRunner( script=train_script, script_args=f"--data_root_dir {data_output_dir}", - params_exchange_format=ExchangeFormat.RAW, # kmeans requires raw values only rather than PyTorch Tensors (the default) + framework=FrameworkType.RAW, # kmeans requires raw values only rather than PyTorch Tensors (the default) ) job.to(executor, f"site-{i+1}") # HIGGs data splitter assumes site names start from 1 diff --git a/examples/advanced/job_api/tf/tf_fl_script_runner_cifar10.py b/examples/advanced/job_api/tf/tf_fl_script_runner_cifar10.py index b2d809a2d4..5d1346665b 100644 --- a/examples/advanced/job_api/tf/tf_fl_script_runner_cifar10.py +++ b/examples/advanced/job_api/tf/tf_fl_script_runner_cifar10.py @@ -23,7 +23,7 @@ from nvflare import FedJob from nvflare.app_common.widgets.intime_model_selector import IntimeModelSelector from nvflare.app_opt.tf.job_config.model import TFModel -from nvflare.job_config.script_runner import ScriptRunner +from nvflare.job_config.script_runner import FrameworkType, ScriptRunner gpu_devices = tf.config.experimental.list_physical_devices("GPU") for device in gpu_devices: @@ -159,7 +159,9 @@ # Add clients for i, train_idx_path in enumerate(train_idx_paths): curr_task_script_args = task_script_args + f" --train_idx_path {train_idx_path}" - executor = ScriptRunner(script=train_script, script_args=curr_task_script_args) + executor = ScriptRunner( + script=train_script, script_args=curr_task_script_args, framework=FrameworkType.TENSORFLOW + ) job.to(executor, f"site-{i+1}") # Can export current job to folder. diff --git a/examples/getting_started/sklearn/kmeans_script_runner_higgs.py b/examples/getting_started/sklearn/kmeans_script_runner_higgs.py index 592b934d8f..d1aaaf6b1d 100644 --- a/examples/getting_started/sklearn/kmeans_script_runner_higgs.py +++ b/examples/getting_started/sklearn/kmeans_script_runner_higgs.py @@ -24,8 +24,7 @@ from nvflare.app_common.widgets.intime_model_selector import IntimeModelSelector from nvflare.app_common.workflows.scatter_and_gather import ScatterAndGather from nvflare.app_opt.sklearn.joblib_model_param_persistor import JoblibModelParamPersistor -from nvflare.client.config import ExchangeFormat -from nvflare.job_config.script_runner import ScriptRunner +from nvflare.job_config.script_runner import FrameworkType, ScriptRunner preprocess = True # if False, assume data is already preprocessed and split @@ -146,7 +145,7 @@ def split_higgs(input_data_path, input_header_path, output_dir, site_num, sample executor = ScriptRunner( script=train_script, script_args=f"--data_root_dir {data_output_dir}", - params_exchange_format=ExchangeFormat.RAW, # kmeans requires raw values only rather than PyTorch Tensors (the default) + framework=FrameworkType.RAW, # kmeans requires raw values only rather than PyTorch Tensors (the default) ) job.to(executor, f"site-{i+1}") # HIGGs data splitter assumes site names start from 1 diff --git a/examples/getting_started/tf/nvflare_tf_getting_started.ipynb b/examples/getting_started/tf/nvflare_tf_getting_started.ipynb index f5e06707c3..7a4ca8fd30 100644 --- a/examples/getting_started/tf/nvflare_tf_getting_started.ipynb +++ b/examples/getting_started/tf/nvflare_tf_getting_started.ipynb @@ -262,7 +262,7 @@ "outputs": [], "source": [ "from nvflare import FedJob\n", - "from nvflare.job_config.script_runner import ScriptRunner\n", + "from nvflare.job_config.script_runner import FrameworkType, ScriptRunner\n", "from nvflare.app_common.workflows.fedavg import FedAvg\n", "\n", "job = FedJob(name=\"cifar10_tf_fedavg\")" @@ -363,12 +363,10 @@ "metadata": {}, "outputs": [], "source": [ - "from nvflare.client.config import ExchangeFormat\n", - "\n", "for i in range(n_clients):\n", " executor = ScriptRunner(\n", " script=\"src/cifar10_tf_fl.py\", script_args=\"\", # f\"--batch_size 32 --data_path /tmp/data/site-{i}\"\n", - " params_exchange_format=ExchangeFormat.NUMPY,\n", + " framework=FrameworkType.TENSORFLOW,\n", " )\n", " job.to(executor, f\"site-{i+1}\")" ] diff --git a/examples/getting_started/tf/tf_fl_script_runner_cifar10.py b/examples/getting_started/tf/tf_fl_script_runner_cifar10.py index b2d809a2d4..5d1346665b 100644 --- a/examples/getting_started/tf/tf_fl_script_runner_cifar10.py +++ b/examples/getting_started/tf/tf_fl_script_runner_cifar10.py @@ -23,7 +23,7 @@ from nvflare import FedJob from nvflare.app_common.widgets.intime_model_selector import IntimeModelSelector from nvflare.app_opt.tf.job_config.model import TFModel -from nvflare.job_config.script_runner import ScriptRunner +from nvflare.job_config.script_runner import FrameworkType, ScriptRunner gpu_devices = tf.config.experimental.list_physical_devices("GPU") for device in gpu_devices: @@ -159,7 +159,9 @@ # Add clients for i, train_idx_path in enumerate(train_idx_paths): curr_task_script_args = task_script_args + f" --train_idx_path {train_idx_path}" - executor = ScriptRunner(script=train_script, script_args=curr_task_script_args) + executor = ScriptRunner( + script=train_script, script_args=curr_task_script_args, framework=FrameworkType.TENSORFLOW + ) job.to(executor, f"site-{i+1}") # Can export current job to folder. diff --git a/examples/hello-world/hello-cyclic/README.md b/examples/hello-world/hello-cyclic/README.md index bec40d40b2..d947363b0a 100644 --- a/examples/hello-world/hello-cyclic/README.md +++ b/examples/hello-world/hello-cyclic/README.md @@ -29,7 +29,7 @@ bash ./prepare_data.sh Run the script using the job API to create the job and run it with the simulator: ``` -python3 cyclic_script_runner_hello-cyclic.py +python3 cyclic_script_runner.py ``` ### 3. Access the logs and results diff --git a/examples/hello-world/hello-cyclic/cyclic_script_runner_hello-cyclic.py b/examples/hello-world/hello-cyclic/cyclic_script_runner.py similarity index 85% rename from examples/hello-world/hello-cyclic/cyclic_script_runner_hello-cyclic.py rename to examples/hello-world/hello-cyclic/cyclic_script_runner.py index da06776c6e..1b2f82652c 100644 --- a/examples/hello-world/hello-cyclic/cyclic_script_runner_hello-cyclic.py +++ b/examples/hello-world/hello-cyclic/cyclic_script_runner.py @@ -16,9 +16,8 @@ from nvflare import FedJob from nvflare.app_common.workflows.cyclic import Cyclic -from nvflare.app_opt.pt.job_config.model import PTModel -from nvflare.client.config import ExchangeFormat -from nvflare.job_config.script_runner import ScriptRunner +from nvflare.app_opt.tf.job_config.model import TFModel +from nvflare.job_config.script_runner import FrameworkType, ScriptRunner if __name__ == "__main__": n_clients = 2 @@ -35,14 +34,14 @@ job.to(controller, "server") # Define the initial global model and send to server - job.to(PTModel(Net()), "server") + job.to(TFModel(Net()), "server") # Add clients for i in range(n_clients): executor = ScriptRunner( script=train_script, script_args="", # f"--batch_size 32 --data_path /tmp/data/site-{i}" - params_exchange_format=ExchangeFormat.NUMPY, + framework=FrameworkType.TENSORFLOW, ) job.to(executor, f"site-{i+1}") diff --git a/examples/hello-world/hello-fedavg-numpy/fedavg_script_runner_hello-numpy.py b/examples/hello-world/hello-fedavg-numpy/fedavg_script_runner_hello-numpy.py index 6fb676f452..ccb2929f54 100644 --- a/examples/hello-world/hello-fedavg-numpy/fedavg_script_runner_hello-numpy.py +++ b/examples/hello-world/hello-fedavg-numpy/fedavg_script_runner_hello-numpy.py @@ -15,8 +15,7 @@ from nvflare import FedJob from nvflare.app_common.widgets.intime_model_selector import IntimeModelSelector from nvflare.app_common.workflows.fedavg import FedAvg -from nvflare.client.config import ExchangeFormat -from nvflare.job_config.script_runner import ScriptRunner +from nvflare.job_config.script_runner import FrameworkType, ScriptRunner if __name__ == "__main__": n_clients = 2 @@ -36,7 +35,7 @@ # Add clients for i in range(n_clients): - executor = ScriptRunner(script=train_script, script_args="", params_exchange_format=ExchangeFormat.NUMPY) + executor = ScriptRunner(script=train_script, script_args="", framework=FrameworkType.NUMPY) job.to(executor, f"site-{i+1}") # job.export_job("/tmp/nvflare/jobs/job_config") diff --git a/examples/hello-world/hello-fedavg-numpy/hello-fedavg-numpy_getting_started.ipynb b/examples/hello-world/hello-fedavg-numpy/hello-fedavg-numpy_getting_started.ipynb index 4fc9b9f4d7..11564a57c0 100644 --- a/examples/hello-world/hello-fedavg-numpy/hello-fedavg-numpy_getting_started.ipynb +++ b/examples/hello-world/hello-fedavg-numpy/hello-fedavg-numpy_getting_started.ipynb @@ -194,7 +194,7 @@ "source": [ "from nvflare import FedJob\n", "from nvflare.app_common.workflows.fedavg import FedAvg\n", - "from nvflare.job_config.script_runner import ScriptRunner\n", + "from nvflare.job_config.script_runner import FrameworkType, ScriptRunner\n", "\n", "\n", "job = FedJob(name=\"hello-fedavg-numpy\")" @@ -266,13 +266,11 @@ "metadata": {}, "outputs": [], "source": [ - "from nvflare.client.config import ExchangeFormat\n", - "\n", "train_script = \"src/hello-numpy_fl.py\"\n", "\n", "for i in range(n_clients):\n", " executor = ScriptRunner(\n", - " script=train_script, script_args=\"\", params_exchange_format=ExchangeFormat.NUMPY\n", + " script=train_script, script_args=\"\", framework=FrameworkType.NUMPY\n", " )\n", " job.to(executor, f\"site-{i}\")" ] diff --git a/examples/hello-world/hello-pt/README.md b/examples/hello-world/hello-pt/README.md index 92f3cabc12..bb12b9cdb5 100644 --- a/examples/hello-world/hello-pt/README.md +++ b/examples/hello-world/hello-pt/README.md @@ -31,7 +31,7 @@ pip3 install -r requirements.txt Run the script using the job API to create the job and run it with the simulator: ``` -python3 fedavg_script_runner_hello-pt.py +python3 fedavg_script_runner_pt.py ``` ### 3. Access the logs and results diff --git a/examples/hello-world/hello-pt/fedavg_script_runner_hello-pt.py b/examples/hello-world/hello-pt/fedavg_script_runner_pt.py similarity index 100% rename from examples/hello-world/hello-pt/fedavg_script_runner_hello-pt.py rename to examples/hello-world/hello-pt/fedavg_script_runner_pt.py diff --git a/examples/hello-world/hello-tf/README.md b/examples/hello-world/hello-tf/README.md index 34c2e5b230..d5ab0b4bfc 100644 --- a/examples/hello-world/hello-tf/README.md +++ b/examples/hello-world/hello-tf/README.md @@ -27,7 +27,7 @@ pip3 install tensorflow Run the script using the job API to create the job and run it with the simulator: ``` -python3 fedavg_script_runner_hello-tf.py +python3 fedavg_script_runner_tf.py ``` ### 3. Access the logs and results diff --git a/examples/hello-world/hello-tf/fedavg_script_runner_hello-tf.py b/examples/hello-world/hello-tf/fedavg_script_runner_tf.py similarity index 83% rename from examples/hello-world/hello-tf/fedavg_script_runner_hello-tf.py rename to examples/hello-world/hello-tf/fedavg_script_runner_tf.py index a146e77de1..1770a92b42 100644 --- a/examples/hello-world/hello-tf/fedavg_script_runner_hello-tf.py +++ b/examples/hello-world/hello-tf/fedavg_script_runner_tf.py @@ -15,7 +15,7 @@ from src.tf_net import TFNet from nvflare.app_opt.tf.job_config.fed_avg import FedAvgJob -from nvflare.job_config.script_runner import ScriptRunner +from nvflare.job_config.script_runner import FrameworkType, ScriptRunner if __name__ == "__main__": n_clients = 2 @@ -27,7 +27,9 @@ # Add clients for i in range(n_clients): executor = ScriptRunner( - script=train_script, script_args="" # f"--batch_size 32 --data_path /tmp/data/site-{i}" + script=train_script, + script_args="", # f"--batch_size 32 --data_path /tmp/data/site-{i}" + framework=FrameworkType.TENSORFLOW, ) job.to(executor, f"site-{i+1}")