diff --git a/job_templates/cyclic_cc_pt/config_fed_client.conf b/job_templates/cyclic_cc_pt/config_fed_client.conf index 07d9f70dc2..3b4980061f 100644 --- a/job_templates/cyclic_cc_pt/config_fed_client.conf +++ b/job_templates/cyclic_cc_pt/config_fed_client.conf @@ -98,9 +98,12 @@ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/cyclic_cc_pt/info.md b/job_templates/cyclic_cc_pt/info.md index af290d5306..5482a9863e 100644 --- a/job_templates/cyclic_cc_pt/info.md +++ b/job_templates/cyclic_cc_pt/info.md @@ -8,4 +8,4 @@ executor_type = "launcher_executor" contributor = "NVIDIA" init_publish_date = "2023-10-13" - last_updated_date = "2023-11-06" # yyyy-mm-dd + last_updated_date = "2023-11-07" # yyyy-mm-dd diff --git a/job_templates/cyclic_pt/config_fed_client.conf b/job_templates/cyclic_pt/config_fed_client.conf index 94db993d38..b6f067a6e6 100644 --- a/job_templates/cyclic_pt/config_fed_client.conf +++ b/job_templates/cyclic_pt/config_fed_client.conf @@ -77,9 +77,12 @@ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/cyclic_pt/info.md b/job_templates/cyclic_pt/info.md index 2e5deeb48b..1f43015f24 100644 --- a/job_templates/cyclic_pt/info.md +++ b/job_templates/cyclic_pt/info.md @@ -8,4 +8,4 @@ executor_type = "launcher_executor" contributor = "NVIDIA" init_publish_date = "2023-10-13" - last_updated_date = "2023-11-06" # yyyy-mm-dd + last_updated_date = "2023-11-07" # yyyy-mm-dd diff --git a/job_templates/sag_nemo/app_1/config_fed_client.conf b/job_templates/sag_nemo/app_1/config_fed_client.conf index bc3779f264..90deb8db1d 100644 --- a/job_templates/sag_nemo/app_1/config_fed_client.conf +++ b/job_templates/sag_nemo/app_1/config_fed_client.conf @@ -80,9 +80,12 @@ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/sag_nemo/app_2/config_fed_client.conf b/job_templates/sag_nemo/app_2/config_fed_client.conf index cf8ae70601..88149dfcc5 100644 --- a/job_templates/sag_nemo/app_2/config_fed_client.conf +++ b/job_templates/sag_nemo/app_2/config_fed_client.conf @@ -80,9 +80,12 @@ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/sag_nemo/app_3/config_fed_client.conf b/job_templates/sag_nemo/app_3/config_fed_client.conf index e19cbfd51a..94b5c50229 100644 --- a/job_templates/sag_nemo/app_3/config_fed_client.conf +++ b/job_templates/sag_nemo/app_3/config_fed_client.conf @@ -80,9 +80,12 @@ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/sag_nemo/info.md b/job_templates/sag_nemo/info.md index ab876c8b7f..7ae42446c9 100644 --- a/job_templates/sag_nemo/info.md +++ b/job_templates/sag_nemo/info.md @@ -8,4 +8,4 @@ executor_type = "launcher_executor" contributor = "NVIDIA" init_publish_date = "2023-10-24" - last_updated_date = "2023-11-06" # yyyy-mm-dd + last_updated_date = "2023-11-07" # yyyy-mm-dd diff --git a/job_templates/sag_pt/config_fed_client.conf b/job_templates/sag_pt/config_fed_client.conf index 3b8f272112..11a2a092df 100644 --- a/job_templates/sag_pt/config_fed_client.conf +++ b/job_templates/sag_pt/config_fed_client.conf @@ -77,9 +77,12 @@ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/sag_pt/info.md b/job_templates/sag_pt/info.md index f9dd1d84c7..464a4c2e16 100644 --- a/job_templates/sag_pt/info.md +++ b/job_templates/sag_pt/info.md @@ -8,4 +8,4 @@ executor_type = "launcher_executor" contributor = "NVIDIA" init_publish_date = "2023-09-11" - last_updated_date = "2023-11-06" # yyyy-mm-dd + last_updated_date = "2023-11-07" # yyyy-mm-dd diff --git a/job_templates/sag_pt_deploy_map/app_1/config_fed_client.conf b/job_templates/sag_pt_deploy_map/app_1/config_fed_client.conf index 3b8f272112..11a2a092df 100644 --- a/job_templates/sag_pt_deploy_map/app_1/config_fed_client.conf +++ b/job_templates/sag_pt_deploy_map/app_1/config_fed_client.conf @@ -77,9 +77,12 @@ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/sag_pt_deploy_map/app_2/config_fed_client.conf b/job_templates/sag_pt_deploy_map/app_2/config_fed_client.conf index 3b8f272112..11a2a092df 100644 --- a/job_templates/sag_pt_deploy_map/app_2/config_fed_client.conf +++ b/job_templates/sag_pt_deploy_map/app_2/config_fed_client.conf @@ -77,9 +77,12 @@ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/sag_pt_deploy_map/info.md b/job_templates/sag_pt_deploy_map/info.md index ef169bde8b..45e27a9502 100644 --- a/job_templates/sag_pt_deploy_map/info.md +++ b/job_templates/sag_pt_deploy_map/info.md @@ -8,4 +8,4 @@ executor_type = "launcher_executor" contributor = "NVIDIA" init_publish_date = "2023-09-11" - last_updated_date = "2023-11-02" # yyyy-mm-dd + last_updated_date = "2023-11-07" # yyyy-mm-dd diff --git a/job_templates/sag_pt_gnn_supervised/app_1/config_fed_client.conf b/job_templates/sag_pt_gnn_supervised/app_1/config_fed_client.conf index 291ba5ea27..8c7b510bda 100644 --- a/job_templates/sag_pt_gnn_supervised/app_1/config_fed_client.conf +++ b/job_templates/sag_pt_gnn_supervised/app_1/config_fed_client.conf @@ -77,9 +77,12 @@ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/sag_pt_gnn_supervised/app_2/config_fed_client.conf b/job_templates/sag_pt_gnn_supervised/app_2/config_fed_client.conf index 291ba5ea27..8c7b510bda 100644 --- a/job_templates/sag_pt_gnn_supervised/app_2/config_fed_client.conf +++ b/job_templates/sag_pt_gnn_supervised/app_2/config_fed_client.conf @@ -77,9 +77,12 @@ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/sag_pt_gnn_supervised/info.md b/job_templates/sag_pt_gnn_supervised/info.md index 18f234ebd5..97dcaa880f 100644 --- a/job_templates/sag_pt_gnn_supervised/info.md +++ b/job_templates/sag_pt_gnn_supervised/info.md @@ -8,4 +8,4 @@ executor_type = "launcher_executor" contributor = "NVIDIA" init_publish_date = "2023-10-31" - last_updated_date = "2023-11-06" # yyyy-mm-dd + last_updated_date = "2023-11-07" # yyyy-mm-dd diff --git a/job_templates/sag_pt_gnn_unsupervised/app_1/config_fed_client.conf b/job_templates/sag_pt_gnn_unsupervised/app_1/config_fed_client.conf index 291ba5ea27..8c7b510bda 100644 --- a/job_templates/sag_pt_gnn_unsupervised/app_1/config_fed_client.conf +++ b/job_templates/sag_pt_gnn_unsupervised/app_1/config_fed_client.conf @@ -77,9 +77,12 @@ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/sag_pt_gnn_unsupervised/app_2/config_fed_client.conf b/job_templates/sag_pt_gnn_unsupervised/app_2/config_fed_client.conf index 291ba5ea27..8c7b510bda 100644 --- a/job_templates/sag_pt_gnn_unsupervised/app_2/config_fed_client.conf +++ b/job_templates/sag_pt_gnn_unsupervised/app_2/config_fed_client.conf @@ -77,9 +77,12 @@ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/sag_pt_gnn_unsupervised/info.md b/job_templates/sag_pt_gnn_unsupervised/info.md index 36f1a1b5a3..247ca49e5c 100644 --- a/job_templates/sag_pt_gnn_unsupervised/info.md +++ b/job_templates/sag_pt_gnn_unsupervised/info.md @@ -8,4 +8,4 @@ executor_type = "launcher_executor" contributor = "NVIDIA" init_publish_date = "2023-10-31" - last_updated_date = "2023-11-06" # yyyy-mm-dd + last_updated_date = "2023-11-07" # yyyy-mm-dd diff --git a/job_templates/sag_tf/config_fed_client.conf b/job_templates/sag_tf/config_fed_client.conf index f0a97e2f5d..a427cb5ddd 100644 --- a/job_templates/sag_tf/config_fed_client.conf +++ b/job_templates/sag_tf/config_fed_client.conf @@ -61,7 +61,7 @@ components = [ { id = "from_nvflare" # NVFlare server side expects a flattened dict, where each key maps to one array - # while Keras layer.get_weights() returns a hierachical dict, where each key might map to a list of arrays + # while Keras layer.get_weights() returns a hierarchical dict, where each key might map to a list of arrays # for example: {"layer0": [weights_array, bias_array]}, so we need to do the conversion # please refer to nvflare.app_opt.tf.utils for detailed implementation # convert flattened numpy dict back to a dict of {layer_name: layer.get_weights()} @@ -80,9 +80,12 @@ components = [ path = "nvflare.fuel.utils.pipe.file_pipe.FilePipe" args { + # Mode of the endpoint. A pipe has two endpoints. + # An endpoint can be either the one that initiates communication or the one listening. + # PASSIVE is the one listening. mode = "PASSIVE" - # root_path: is the directory location of the parameters exchange. + # root_path: is the directory location of the data exchange. # If empty string, it will be set to the app_dir of the running job. # You can also set it to an absolute path in your system. root_path = "" diff --git a/job_templates/sag_tf/info.md b/job_templates/sag_tf/info.md index 144442a46f..f317003535 100644 --- a/job_templates/sag_tf/info.md +++ b/job_templates/sag_tf/info.md @@ -8,4 +8,4 @@ executor_type = "launcher_executor" contributor = "NVIDIA" init_publish_date = "2023-09-11" - last_updated_date = "2023-11-06" # yyyy-mm-dd + last_updated_date = "2023-11-07" # yyyy-mm-dd diff --git a/job_templates/swarm_cse_pt/config_fed_client.conf b/job_templates/swarm_cse_pt/config_fed_client.conf index 75c6355f19..65b1a92748 100755 --- a/job_templates/swarm_cse_pt/config_fed_client.conf +++ b/job_templates/swarm_cse_pt/config_fed_client.conf @@ -21,14 +21,14 @@ executors = [ # All tasks prefixed with swarm_ are routed to SwarmClientController tasks = ["swarm_*"] executor { - # client-side controller for training and logic and aggregration management + # client-side controller for training and logic and aggregation management path = "nvflare.app_common.ccwf.SwarmClientController" args { # train task must be implemented by Executor learn_task_name = "train" # how long to wait for current learn task before timing out the gathering learn_task_timeout = 600 - # ids must map to corresponding componenets + # ids must map to corresponding components persistor_id = "persistor" aggregator_id = "aggregator" shareable_generator_id = "shareable_generator" diff --git a/nvflare/fuel/utils/pipe/file_pipe.py b/nvflare/fuel/utils/pipe/file_pipe.py index a850ec8d55..f0ca8db2b5 100644 --- a/nvflare/fuel/utils/pipe/file_pipe.py +++ b/nvflare/fuel/utils/pipe/file_pipe.py @@ -29,7 +29,11 @@ def __init__(self, mode: Mode, root_path: str, file_check_interval=0.1): """Implementation of communication through the file system. Args: - root_path: root path + mode (Mode): Mode of the endpoint. A pipe has two endpoints. + An endpoint can be either the one that initiates communication or the one listening. + root_path (str): root path for this file pipe, folders and files will be created under this root_path + for communication. + file_check_interval (float): how often should to check the file exists. """ super().__init__(mode=mode) check_positive_number("file_check_interval", file_check_interval)