From 79629aa07691e3bd376bc40e33d464749b833cb8 Mon Sep 17 00:00:00 2001 From: Yuhong Wen Date: Tue, 17 Dec 2024 14:19:50 -0500 Subject: [PATCH 1/3] Added docker job launcher provistion. --- .../app_opt/job_launcher/docker_launcher.py | 3 +- nvflare/lighter/constants.py | 5 +- nvflare/lighter/ctx.py | 1 + nvflare/lighter/dummy_project.yml | 9 ++ nvflare/lighter/impl/docker.py | 70 ++++++++++++++- nvflare/lighter/impl/static_file.py | 10 +++ nvflare/lighter/templates/master_template.yml | 89 ++++++++++--------- 7 files changed, 140 insertions(+), 47 deletions(-) diff --git a/nvflare/app_opt/job_launcher/docker_launcher.py b/nvflare/app_opt/job_launcher/docker_launcher.py index 48627db81a..fde29e9ed7 100644 --- a/nvflare/app_opt/job_launcher/docker_launcher.py +++ b/nvflare/app_opt/job_launcher/docker_launcher.py @@ -182,9 +182,10 @@ def get_command(self, job_meta, fl_ctx) -> (str, str): class ClientDockerJobLauncher(DockerJobLauncher): def get_command(self, job_meta, fl_ctx) -> (str, str): job_id = job_meta.get(JobConstants.JOB_ID) + client_name = fl_ctx.get_identity_name() command = generate_client_command(job_meta, fl_ctx) - return f"client-{job_id}", command + return f"{client_name}-{job_id}", command class ServerDockerJobLauncher(DockerJobLauncher): diff --git a/nvflare/lighter/constants.py b/nvflare/lighter/constants.py index 97b18d0595..95283c7433 100644 --- a/nvflare/lighter/constants.py +++ b/nvflare/lighter/constants.py @@ -43,6 +43,7 @@ class PropKey: OVERSEER_END_POINT = "overseer_end_point" ADMIN_PORT = "admin_port" FED_LEARN_PORT = "fed_learn_port" + COMMUNICATION_PORT = "communication_port" class CtxKey(WorkDir, PropKey): @@ -87,6 +88,7 @@ class TemplateSectionKey: SUB_START_SH = "sub_start_sh" STOP_FL_SH = "stop_fl_sh" LOG_CONFIG = "log_config" + COMM_CONFIG = "comm_config" LOCAL_SERVER_RESOURCES = "local_server_resources" LOCAL_CLIENT_RESOURCES = "local_client_resources" SAMPLE_PRIVACY = "sample_privacy" @@ -110,11 +112,12 @@ class ProvFileName: START_SH = "start.sh" SUB_START_SH = "sub_start.sh" PRIVILEGE_YML = "privilege.yml" - DOCKER_SH = "docker.sh" + DOCKER_SH = "start_docker.sh" GUNICORN_CONF_PY = "gunicorn.conf.py" FED_SERVER_JSON = "fed_server.json" FED_CLIENT_JSON = "fed_client.json" STOP_FL_SH = "stop_fl.sh" + COMM_CONFIG = "comm_config.json" LOG_CONFIG_DEFAULT = "log.config.default" RESOURCES_JSON_DEFAULT = "resources.json.default" PRIVACY_JSON_SAMPLE = "privacy.json.sample" diff --git a/nvflare/lighter/ctx.py b/nvflare/lighter/ctx.py index f55cdfd303..34bdba19ce 100644 --- a/nvflare/lighter/ctx.py +++ b/nvflare/lighter/ctx.py @@ -43,6 +43,7 @@ def __init__(self, workspace_root_dir: str, project: Project): fed_learn_port = server.get_prop(PropKey.FED_LEARN_PORT, 8002) self[CtxKey.FED_LEARN_PORT] = fed_learn_port self[CtxKey.SERVER_NAME] = server.name + self[CtxKey.COMMUNICATION_PORT] = server.get_prop(PropKey.COMMUNICATION_PORT, 8005) def get_project(self): return self.get(CtxKey.PROJECT) diff --git a/nvflare/lighter/dummy_project.yml b/nvflare/lighter/dummy_project.yml index 69ca96fdcc..469faaeac0 100644 --- a/nvflare/lighter/dummy_project.yml +++ b/nvflare/lighter/dummy_project.yml @@ -11,6 +11,7 @@ participants: org: nvidia fed_learn_port: 8002 admin_port: 8003 + communication_port: 8005 - name: site-1 type: client org: nvidia @@ -19,6 +20,7 @@ participants: # as a server for 3rd-party integration. # The value must be a hostname that the external trainer can reach via the network. # listening_host: site-1-lh + communication_port: 8006 - name: site-2 type: client org: nvidia @@ -65,5 +67,12 @@ builders: args: sp_end_point: server1:8002:8003 + docker_image: localhost/nvflare:0.0.1 + + - path: nvflare.lighter.impl.docker.DockerBuilder + args: + docker_image: localhost/nvflare:0.0.1 + base_image: python:3.10 + requirements_file: docker_compose_requirements.txt - path: nvflare.lighter.impl.cert.CertBuilder - path: nvflare.lighter.impl.signature.SignatureBuilder diff --git a/nvflare/lighter/impl/docker.py b/nvflare/lighter/impl/docker.py index 8773b64dc7..fc4def738b 100644 --- a/nvflare/lighter/impl/docker.py +++ b/nvflare/lighter/impl/docker.py @@ -13,18 +13,24 @@ # limitations under the License. import copy +import json import os import shutil import yaml -from nvflare.lighter.constants import CtxKey, ProvFileName, TemplateSectionKey +from nvflare.app_opt.job_launcher.docker_launcher import ClientDockerJobLauncher, ServerDockerJobLauncher +from nvflare.lighter import utils +from nvflare.lighter.constants import CtxKey, PropKey, ProvFileName, TemplateSectionKey from nvflare.lighter.spec import Builder, Project, ProvisionContext class DockerBuilder(Builder): - def __init__(self, base_image="python:3.8", requirements_file="requirements.txt"): + def __init__( + self, docker_image="nvflare-docker:0.0.1", base_image="python:3.8", requirements_file="requirements.txt" + ): """Build docker compose file.""" + self.docker_image = docker_image self.base_image = base_image self.requirements_file = requirements_file self.services = {} @@ -57,7 +63,31 @@ def _build_server(self, server, ctx: ProvisionContext): info_dict["container_name"] = server.name self.services[server.name] = info_dict - def _build_client(self, client): + # local folder creation + dest_dir = ctx.get_local_dir(server) + with open(os.path.join(dest_dir, ProvFileName.RESOURCES_JSON_DEFAULT), "rt") as f: + resources = json.load(f) + resources["components"].append( + { + "id": "docker_launcher", + "path": ServerDockerJobLauncher().__module__ + "." + "ServerDockerJobLauncher", + "args": {}, + } + ) + utils.write(os.path.join(dest_dir, ProvFileName.RESOURCES_JSON_DEFAULT), json.dumps(resources, indent=4), "t") + + communication_port = ctx.get(CtxKey.COMMUNICATION_PORT) + if communication_port: + replacement_dict = {"comm_host_name": "server-parent", "communication_port": communication_port} + ctx.build_from_template( + dest_dir, + TemplateSectionKey.COMM_CONFIG, + ProvFileName.COMM_CONFIG, + replacement=replacement_dict, + exe=True, + ) + + def _build_client(self, client, ctx: ProvisionContext): info_dict = copy.deepcopy(self.services["__flclient__"]) info_dict["volumes"] = [f"./{client.name}:" + "${WORKSPACE}"] info_dict["build"] = "nvflare_compose" @@ -71,6 +101,30 @@ def _build_client(self, client): info_dict["container_name"] = client.name self.services[client.name] = info_dict + # local folder creation + dest_dir = ctx.get_local_dir(client) + with open(os.path.join(dest_dir, ProvFileName.RESOURCES_JSON_DEFAULT), "rt") as f: + resources = json.load(f) + resources["components"].append( + { + "id": "docker_launcher", + "path": ClientDockerJobLauncher().__module__ + "." + "ClientDockerJobLauncher", + "args": {}, + } + ) + utils.write(os.path.join(dest_dir, ProvFileName.RESOURCES_JSON_DEFAULT), json.dumps(resources, indent=4), "t") + + communication_port = client.get_prop(PropKey.COMMUNICATION_PORT) + if communication_port: + replacement_dict = {"comm_host_name": client.name + "-parent", "communication_port": communication_port} + ctx.build_from_template( + dest_dir, + TemplateSectionKey.COMM_CONFIG, + ProvFileName.COMM_CONFIG, + replacement=replacement_dict, + exe=True, + ) + def build(self, project: Project, ctx: ProvisionContext): compose = ctx.yaml_load_template_section(TemplateSectionKey.COMPOSE_YAML) self.services = compose.get("services") @@ -83,7 +137,7 @@ def build(self, project: Project, ctx: ProvisionContext): self._build_server(server, ctx) for client in project.get_clients(): - self._build_client(client) + self._build_client(client, ctx) self.services.pop("__overseer__", None) self.services.pop("__flserver__", None) @@ -101,6 +155,14 @@ def build(self, project: Project, ctx: ProvisionContext): with open(os.path.join(compose_build_dir, ProvFileName.DOCKERFILE), "wt") as f: f.write(f"FROM {self.base_image}\n") f.write(ctx.get_template_section(TemplateSectionKey.DOCKERFILE)) + replacement_dict = {"image": self.docker_image} + ctx.build_from_template( + compose_build_dir, + "docker_build_sh", + "docker_build.sh", + replacement=replacement_dict, + exe=True, + ) try: shutil.copyfile(self.requirements_file, os.path.join(compose_build_dir, ProvFileName.REQUIREMENTS_TXT)) except Exception: diff --git a/nvflare/lighter/impl/static_file.py b/nvflare/lighter/impl/static_file.py index 742f2eaac2..44646362a6 100644 --- a/nvflare/lighter/impl/static_file.py +++ b/nvflare/lighter/impl/static_file.py @@ -112,10 +112,12 @@ def _build_server(self, server: Participant, ctx: ProvisionContext): server_0["name"] = project.name admin_port = ctx.get(CtxKey.ADMIN_PORT) fed_learn_port = ctx.get(CtxKey.FED_LEARN_PORT) + communication_port = ctx.get(CtxKey.COMMUNICATION_PORT) server_0["service"]["target"] = f"{server.name}:{fed_learn_port}" server_0["service"]["scheme"] = self.scheme server_0["admin_host"] = server.name server_0["admin_port"] = admin_port + server_0["communication_port"] = communication_port self._prepare_overseer_agent(server, config, OverseerRole.SERVER, ctx) utils.write(os.path.join(dest_dir, ProvFileName.FED_SERVER_JSON), json.dumps(config, indent=2), "t") @@ -123,6 +125,7 @@ def _build_server(self, server: Participant, ctx: ProvisionContext): replacement_dict = { "admin_port": admin_port, "fed_learn_port": fed_learn_port, + "communication_port": communication_port, "config_folder": self.config_folder, "docker_image": self.docker_image, "org_name": server.org, @@ -182,7 +185,14 @@ def _build_client(self, client, ctx): config["servers"][0]["service"]["scheme"] = self.scheme config["servers"][0]["name"] = project.name config["servers"][0]["identity"] = server.name # the official identity of the server + admin_port = ctx.get(CtxKey.ADMIN_PORT) + fed_learn_port = ctx.get(CtxKey.FED_LEARN_PORT) + communication_port = client.get_prop(PropKey.COMMUNICATION_PORT) replacement_dict = { + "admin_port": admin_port, + "fed_learn_port": fed_learn_port, + "communication_port": communication_port, + "comm_host_name": client.name + "-parent", "client_name": f"{client.subject}", "config_folder": self.config_folder, "docker_image": self.docker_image, diff --git a/nvflare/lighter/templates/master_template.yml b/nvflare/lighter/templates/master_template.yml index 7b1e51af88..5d257abadb 100644 --- a/nvflare/lighter/templates/master_template.yml +++ b/nvflare/lighter/templates/master_template.yml @@ -230,6 +230,17 @@ local_server_resources: | ] } +comm_config: | + { + "internal": { + "scheme": "tcp", + "resources": { + "host": "{~~comm_host_name~~}", + "port": {~~communication_port~~} + } + } + } + fed_server: | { "format_version": 2, @@ -525,60 +536,44 @@ sub_start_sh: | docker_cln_sh: | #!/usr/bin/env bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - # docker run script for FL client - # local data directory - : ${MY_DATA_DIR:="/home/flclient/data"} - # The syntax above is to set MY_DATA_DIR to /home/flcient/data if this - # environment variable is not set previously. - # Therefore, users can set their own MY_DATA_DIR with - # export MY_DATA_DIR=$SOME_DIRECTORY - # before running docker.sh - - # for all gpus use line below - #GPU2USE='--gpus=all' - # for 2 gpus use line below - #GPU2USE='--gpus=2' - # for specific gpus as gpu#0 and gpu#2 use line below - #GPU2USE='--gpus="device=0,2"' - # to use host network, use line below - NETARG="--net=host" - # FL clients do not need to open ports, so the following line is not needed. - #NETARG="-p 443:443 -p 8003:8003" + # docker run script for FL server DOCKER_IMAGE={~~docker_image~~} echo "Starting docker with $DOCKER_IMAGE" - mode="${1:--r}" - if [ $mode = "-d" ] - then - docker run -d --rm --name={~~client_name~~} $GPU2USE -u $(id -u):$(id -g) \ - -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group -v $DIR/..:/workspace/ \ - -v $MY_DATA_DIR:/data/:ro -w /workspace/ --ipc=host $NETARG $DOCKER_IMAGE \ - /bin/bash -c "python -u -m nvflare.private.fed.app.client.client_train -m /workspace -s fed_client.json --set uid={~~client_name~~} secure_train=true config_folder=config org={~~org_name~~}" + + NETWORK_NAME="nvflare-network" + if docker network ls --filter name=$NETWORK_NAME --format "{{.Name}}" | grep -wq $NETWORK_NAME; then + echo "Network '${NETWORK_NAME}' exists." else - docker run --rm -it --name={~~client_name~~} $GPU2USE -u $(id -u):$(id -g) \ - -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group -v $DIR/..:/workspace/ \ - -v $MY_DATA_DIR:/data/:ro -w /workspace/ --ipc=host $NETARG $DOCKER_IMAGE /bin/bash + docker network create $NETWORK_NAME fi + + docker run --name {~~comm_host_name~~} --network nvflare-network \ + -v $DIR/..:/workspace \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -p {~~communication_port~~}:{~~communication_port~~} \ + -it --rm $DOCKER_IMAGE /bin/bash -c "export NVFL_DOCKER_WORKSPACE=$DIR/..;startup/sub_start.sh \ + {~~client_name~~} server-parent:{~~fed_learn_port~~}:{~~admin_port~~}" docker_svr_sh: | #!/usr/bin/env bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # docker run script for FL server - # to use host network, use line below - NETARG="--net=host" - # or to expose specific ports, use line below - #NETARG="-p {~~admin_port~~}:{~~admin_port~~} -p {~~fed_learn_port~~}:{~~fed_learn_port~~}" DOCKER_IMAGE={~~docker_image~~} echo "Starting docker with $DOCKER_IMAGE" - svr_name="${SVR_NAME:-flserver}" - mode="${1:-r}" - if [ $mode = "-d" ] - then - docker run -d --rm --name=$svr_name -v $DIR/..:/workspace/ -w /workspace \ - --ipc=host $NETARG $DOCKER_IMAGE /bin/bash -c \ - "python -u -m nvflare.private.fed.app.server.server_train -m /workspace -s fed_server.json --set secure_train=true config_folder=config org={~~org_name~~}" + + NETWORK_NAME="nvflare-network" + if docker network ls --filter name=$NETWORK_NAME --format "{{.Name}}" | grep -wq $NETWORK_NAME; then + echo "Network '${NETWORK_NAME}' exists." else - docker run --rm -it --name=$svr_name -v $DIR/..:/workspace/ -w /workspace/ --ipc=host $NETARG $DOCKER_IMAGE /bin/bash + docker network create $NETWORK_NAME fi + + docker run --name server-parent --network nvflare-network \ + -v $DIR/..:/workspace \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -p {~~admin_port~~}:{~~admin_port~~} -p {~~fed_learn_port~~}:{~~fed_learn_port~~} \ + -p {~~communication_port~~}:{~~communication_port~~} \ + -it --rm $DOCKER_IMAGE /bin/bash -c "export NVFL_DOCKER_WORKSPACE=$DIR/..;startup/sub_start.sh" docker_adm_sh: | #!/usr/bin/env bash @@ -648,10 +643,22 @@ compose_yaml: | nvflare_svc_persist: dockerfile: | + RUN mkdir /opt/NVFlare + WORKDIR /opt/NVFlare RUN pip install -U pip RUN pip install nvflare COPY requirements.txt requirements.txt RUN pip install -r requirements.txt + + RUN apt-get update + RUN apt install docker.io -y + WORKDIR /workspace + +docker_build_sh: | + #!/usr/bin/env bash + docker image rm {~~image~~} + docker build -t {~~image~~} -f Dockerfile . + docker push {~~image~~} helm_chart_chart: | apiVersion: v2 From 794533302b4616042856834dbc02da67edf50604 Mon Sep 17 00:00:00 2001 From: Yuhong Wen Date: Tue, 17 Dec 2024 15:29:04 -0500 Subject: [PATCH 2/3] Added docker build provision constants. --- nvflare/lighter/constants.py | 2 ++ nvflare/lighter/impl/docker.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nvflare/lighter/constants.py b/nvflare/lighter/constants.py index 95283c7433..bc29884dfd 100644 --- a/nvflare/lighter/constants.py +++ b/nvflare/lighter/constants.py @@ -78,6 +78,7 @@ class OverseerRole: class TemplateSectionKey: START_SERVER_SH = "start_svr_sh" START_CLIENT_SH = "start_cln_sh" + DOCKER_BUILD_SH = "docker_build_sh" DOCKER_SERVER_SH = "docker_svr_sh" DOCKER_CLIENT_SH = "docker_cln_sh" DOCKER_ADMIN_SH = "docker_adm_sh" @@ -112,6 +113,7 @@ class ProvFileName: START_SH = "start.sh" SUB_START_SH = "sub_start.sh" PRIVILEGE_YML = "privilege.yml" + DOCKER_BUILD_SH = "docker_build.sh" DOCKER_SH = "start_docker.sh" GUNICORN_CONF_PY = "gunicorn.conf.py" FED_SERVER_JSON = "fed_server.json" diff --git a/nvflare/lighter/impl/docker.py b/nvflare/lighter/impl/docker.py index fc4def738b..03409f8af1 100644 --- a/nvflare/lighter/impl/docker.py +++ b/nvflare/lighter/impl/docker.py @@ -158,8 +158,8 @@ def build(self, project: Project, ctx: ProvisionContext): replacement_dict = {"image": self.docker_image} ctx.build_from_template( compose_build_dir, - "docker_build_sh", - "docker_build.sh", + TemplateSectionKey.DOCKER_BUILD_SH, + ProvFileName.DOCKER_BUILD_SH, replacement=replacement_dict, exe=True, ) From 4aec7f3a6d3977415291990829ba2613e5fa05c6 Mon Sep 17 00:00:00 2001 From: Yuhong Wen Date: Thu, 9 Jan 2025 10:36:39 -0500 Subject: [PATCH 3/3] refactored to use docker_comm_port name. --- nvflare/lighter/constants.py | 2 +- nvflare/lighter/ctx.py | 1 - nvflare/lighter/dummy_project.yml | 4 ++-- nvflare/lighter/impl/docker.py | 4 ++-- nvflare/lighter/impl/static_file.py | 5 ++--- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/nvflare/lighter/constants.py b/nvflare/lighter/constants.py index bc29884dfd..45376c9916 100644 --- a/nvflare/lighter/constants.py +++ b/nvflare/lighter/constants.py @@ -43,7 +43,7 @@ class PropKey: OVERSEER_END_POINT = "overseer_end_point" ADMIN_PORT = "admin_port" FED_LEARN_PORT = "fed_learn_port" - COMMUNICATION_PORT = "communication_port" + DOCKER_COMM_PORT = "docker_comm_port" class CtxKey(WorkDir, PropKey): diff --git a/nvflare/lighter/ctx.py b/nvflare/lighter/ctx.py index 34bdba19ce..f55cdfd303 100644 --- a/nvflare/lighter/ctx.py +++ b/nvflare/lighter/ctx.py @@ -43,7 +43,6 @@ def __init__(self, workspace_root_dir: str, project: Project): fed_learn_port = server.get_prop(PropKey.FED_LEARN_PORT, 8002) self[CtxKey.FED_LEARN_PORT] = fed_learn_port self[CtxKey.SERVER_NAME] = server.name - self[CtxKey.COMMUNICATION_PORT] = server.get_prop(PropKey.COMMUNICATION_PORT, 8005) def get_project(self): return self.get(CtxKey.PROJECT) diff --git a/nvflare/lighter/dummy_project.yml b/nvflare/lighter/dummy_project.yml index 469faaeac0..abe21b5d0d 100644 --- a/nvflare/lighter/dummy_project.yml +++ b/nvflare/lighter/dummy_project.yml @@ -11,7 +11,7 @@ participants: org: nvidia fed_learn_port: 8002 admin_port: 8003 - communication_port: 8005 + docker_comm_port: 8005 - name: site-1 type: client org: nvidia @@ -20,7 +20,7 @@ participants: # as a server for 3rd-party integration. # The value must be a hostname that the external trainer can reach via the network. # listening_host: site-1-lh - communication_port: 8006 + docker_comm_port: 8006 - name: site-2 type: client org: nvidia diff --git a/nvflare/lighter/impl/docker.py b/nvflare/lighter/impl/docker.py index 03409f8af1..9a90a3866e 100644 --- a/nvflare/lighter/impl/docker.py +++ b/nvflare/lighter/impl/docker.py @@ -76,7 +76,7 @@ def _build_server(self, server, ctx: ProvisionContext): ) utils.write(os.path.join(dest_dir, ProvFileName.RESOURCES_JSON_DEFAULT), json.dumps(resources, indent=4), "t") - communication_port = ctx.get(CtxKey.COMMUNICATION_PORT) + communication_port = server.get_prop(CtxKey.DOCKER_COMM_PORT) if communication_port: replacement_dict = {"comm_host_name": "server-parent", "communication_port": communication_port} ctx.build_from_template( @@ -114,7 +114,7 @@ def _build_client(self, client, ctx: ProvisionContext): ) utils.write(os.path.join(dest_dir, ProvFileName.RESOURCES_JSON_DEFAULT), json.dumps(resources, indent=4), "t") - communication_port = client.get_prop(PropKey.COMMUNICATION_PORT) + communication_port = client.get_prop(PropKey.DOCKER_COMM_PORT) if communication_port: replacement_dict = {"comm_host_name": client.name + "-parent", "communication_port": communication_port} ctx.build_from_template( diff --git a/nvflare/lighter/impl/static_file.py b/nvflare/lighter/impl/static_file.py index 44646362a6..5f7bddd039 100644 --- a/nvflare/lighter/impl/static_file.py +++ b/nvflare/lighter/impl/static_file.py @@ -112,12 +112,11 @@ def _build_server(self, server: Participant, ctx: ProvisionContext): server_0["name"] = project.name admin_port = ctx.get(CtxKey.ADMIN_PORT) fed_learn_port = ctx.get(CtxKey.FED_LEARN_PORT) - communication_port = ctx.get(CtxKey.COMMUNICATION_PORT) + communication_port = server.get_prop(CtxKey.DOCKER_COMM_PORT) server_0["service"]["target"] = f"{server.name}:{fed_learn_port}" server_0["service"]["scheme"] = self.scheme server_0["admin_host"] = server.name server_0["admin_port"] = admin_port - server_0["communication_port"] = communication_port self._prepare_overseer_agent(server, config, OverseerRole.SERVER, ctx) utils.write(os.path.join(dest_dir, ProvFileName.FED_SERVER_JSON), json.dumps(config, indent=2), "t") @@ -187,7 +186,7 @@ def _build_client(self, client, ctx): config["servers"][0]["identity"] = server.name # the official identity of the server admin_port = ctx.get(CtxKey.ADMIN_PORT) fed_learn_port = ctx.get(CtxKey.FED_LEARN_PORT) - communication_port = client.get_prop(PropKey.COMMUNICATION_PORT) + communication_port = client.get_prop(PropKey.DOCKER_COMM_PORT) replacement_dict = { "admin_port": admin_port, "fed_learn_port": fed_learn_port,