diff --git a/nvflare/apis/fl_constant.py b/nvflare/apis/fl_constant.py index d03e456ad3..d9e0e92315 100644 --- a/nvflare/apis/fl_constant.py +++ b/nvflare/apis/fl_constant.py @@ -245,6 +245,8 @@ class AdminCommandNames(object): SHELL_TAIL = "tail" SHELL_GREP = "grep" APP_COMMAND = "app_command" + CONFIGURE_JOB_LOG = "configure_job_log" + CONFIGURE_SITE_LOG = "configure_site_log" class ServerCommandNames(object): @@ -263,6 +265,7 @@ class ServerCommandNames(object): HANDLE_DEAD_JOB = "handle_dead_job" SERVER_STATE = "server_state" APP_COMMAND = "app_command" + CONFIGURE_JOB_LOG = "configure_job_log" class ServerCommandKey(object): diff --git a/nvflare/fuel/utils/log_utils.py b/nvflare/fuel/utils/log_utils.py index e889d22759..e9848e7846 100644 --- a/nvflare/fuel/utils/log_utils.py +++ b/nvflare/fuel/utils/log_utils.py @@ -20,6 +20,7 @@ from logging import Logger from logging.handlers import RotatingFileHandler +from nvflare.apis.fl_constant import WorkspaceConstants from nvflare.apis.workspace import Workspace @@ -262,6 +263,46 @@ def apply_log_config(dict_config, dir_path: str = "", file_prefix: str = ""): logging.config.dictConfig(dict_config) +def dynamic_log_config(config: str, workspace: Workspace, job_id: str = None): + # Dynamically configure log given a config (filepath, levelname, levelnumber, 'reload'), apply the config to the proper locations. + if not isinstance(config, str): + raise ValueError( + f"Unsupported config type. Expect config to be string filepath, levelname, levelnumber, or 'reload' but got {type(config)}" + ) + + if config == "reload": + config = workspace.get_log_config_file_path() + + if os.path.isfile(config): + # Read confg file + with open(config, "r") as f: + dict_config = json.load(f) + + if job_id: + dir_path = workspace.get_run_dir(job_id) + else: + dir_path = workspace.get_root_dir() + + # overwrite log_config.json of site + with open(os.path.join(workspace.get_site_config_dir(), WorkspaceConstants.LOGGING_CONFIG), "w") as f: + f.write(json.dumps(dict_config)) + + apply_log_config(dict_config, dir_path) + + else: + # Set level of root logger based on levelname or levelnumber + if config.isdigit(): + level = int(config) + if not (0 <= level <= 50): + raise ValueError(f"Invalid logging level: {level}") + else: + level = getattr(logging, config.upper(), None) + if level is None: + raise ValueError(f"Invalid logging level: {config}") + + logging.getLogger().setLevel(level) + + def add_log_file_handler(log_file_name): root_logger = logging.getLogger() main_handler = root_logger.handlers[0] diff --git a/nvflare/job_config/api.py b/nvflare/job_config/api.py index c010a673b6..ae85d32b45 100644 --- a/nvflare/job_config/api.py +++ b/nvflare/job_config/api.py @@ -522,7 +522,9 @@ def export_job(self, job_root: str): self._set_all_apps() self.job.generate_job_config(job_root) - def simulator_run(self, workspace: str, n_clients: int = None, threads: int = None, gpu: str = None): + def simulator_run( + self, workspace: str, n_clients: int = None, threads: int = None, gpu: str = None, log_config: str = None + ): """Run the job with the simulator with the `workspace` using `clients` and `threads`. For end users. @@ -531,6 +533,7 @@ def simulator_run(self, workspace: str, n_clients: int = None, threads: int = No n_clients: number of clients. threads: number of threads. gpu: gpu assignments for simulating clients, comma separated + log_config: log config json file path Returns: @@ -556,6 +559,7 @@ def simulator_run(self, workspace: str, n_clients: int = None, threads: int = No n_clients=n_clients, threads=threads, gpu=gpu, + log_config=log_config, ) def as_id(self, obj: Any) -> str: diff --git a/nvflare/job_config/fed_job_config.py b/nvflare/job_config/fed_job_config.py index 940a773a63..c92fe890e5 100644 --- a/nvflare/job_config/fed_job_config.py +++ b/nvflare/job_config/fed_job_config.py @@ -136,7 +136,7 @@ def generate_job_config(self, job_root): self._generate_meta(job_dir) - def simulator_run(self, workspace, clients=None, n_clients=None, threads=None, gpu=None): + def simulator_run(self, workspace, clients=None, n_clients=None, threads=None, gpu=None, log_config=None): with TemporaryDirectory() as job_root: self.generate_job_config(job_root) @@ -157,6 +157,8 @@ def simulator_run(self, workspace, clients=None, n_clients=None, threads=None, g if gpu: gpu = self._trim_whitespace(gpu) command += " -gpu " + str(gpu) + if log_config: + command += " -l" + str(log_config) new_env = os.environ.copy() process = subprocess.Popen(shlex.split(command, True), preexec_fn=os.setsid, env=new_env) diff --git a/nvflare/private/defs.py b/nvflare/private/defs.py index bb0f98683d..d5f3f96744 100644 --- a/nvflare/private/defs.py +++ b/nvflare/private/defs.py @@ -68,6 +68,7 @@ class TrainingTopic(object): START_JOB = "train.start_job" GET_SCOPES = "train.get_scopes" NOTIFY_JOB_STATUS = "train.notify_job_status" + CONFIGURE_JOB_LOG = "train.configure_job_log" class RequestHeader(object): @@ -96,6 +97,7 @@ class SysCommandTopic(object): SHELL = "sys.shell" REPORT_RESOURCES = "resource_manager.report_resources" REPORT_ENV = "sys.report_env" + CONFIGURE_SITE_LOG = "sys.configure_site_log" class ControlCommandTopic(object): diff --git a/nvflare/private/fed/app/simulator/simulator.py b/nvflare/private/fed/app/simulator/simulator.py index 50a5a56319..6f914be1d9 100644 --- a/nvflare/private/fed/app/simulator/simulator.py +++ b/nvflare/private/fed/app/simulator/simulator.py @@ -29,6 +29,7 @@ def define_simulator_parser(simulator_parser): simulator_parser.add_argument("-c", "--clients", type=str, help="client names list") simulator_parser.add_argument("-t", "--threads", type=int, help="number of parallel running clients") simulator_parser.add_argument("-gpu", "--gpu", type=str, help="list of GPU Device Ids, comma separated") + simulator_parser.add_argument("-l", "--log_config", type=str, help="log config file path") simulator_parser.add_argument("-m", "--max_clients", type=int, default=100, help="max number of clients") simulator_parser.add_argument( "--end_run_for_all", @@ -46,6 +47,7 @@ def run_simulator(simulator_args): n_clients=simulator_args.n_clients, threads=simulator_args.threads, gpu=simulator_args.gpu, + log_config=simulator_args.log_config, max_clients=simulator_args.max_clients, end_run_for_all=simulator_args.end_run_for_all, ) diff --git a/nvflare/private/fed/app/simulator/simulator_runner.py b/nvflare/private/fed/app/simulator/simulator_runner.py index 4896647265..4351af8081 100644 --- a/nvflare/private/fed/app/simulator/simulator_runner.py +++ b/nvflare/private/fed/app/simulator/simulator_runner.py @@ -88,6 +88,7 @@ def __init__( n_clients=None, threads=None, gpu=None, + log_config=None, max_clients=100, end_run_for_all=False, ): @@ -99,6 +100,7 @@ def __init__( self.n_clients = n_clients self.threads = threads self.gpu = gpu + self.log_config = log_config self.max_clients = max_clients self.end_run_for_all = end_run_for_all @@ -126,7 +128,15 @@ def __init__( self.workspace = os.path.join(running_dir, self.workspace) def _generate_args( - self, job_folder: str, workspace: str, clients=None, n_clients=None, threads=None, gpu=None, max_clients=100 + self, + job_folder: str, + workspace: str, + clients=None, + n_clients=None, + threads=None, + gpu=None, + log_config=None, + max_clients=100, ): args = Namespace( job_folder=job_folder, @@ -135,6 +145,7 @@ def _generate_args( n_clients=n_clients, threads=threads, gpu=gpu, + log_config=log_config, max_clients=max_clients, ) args.set = [] @@ -142,7 +153,14 @@ def _generate_args( def setup(self): self.args = self._generate_args( - self.job_folder, self.workspace, self.clients, self.n_clients, self.threads, self.gpu, self.max_clients + self.job_folder, + self.workspace, + self.clients, + self.n_clients, + self.threads, + self.gpu, + self.log_config, + self.max_clients, ) if self.args.clients: @@ -152,14 +170,16 @@ def setup(self): for i in range(self.args.n_clients): self.client_names.append("site-" + str(i + 1)) - log_config_file_path = os.path.join(self.args.workspace, "local", WorkspaceConstants.LOGGING_CONFIG) - if not os.path.isfile(log_config_file_path): - log_config_file_path = os.path.join(os.path.dirname(__file__), WorkspaceConstants.LOGGING_CONFIG) + if self.args.log_config: + log_config_file_path = self.args.log_config + else: + log_config_file_path = os.path.join(self.args.workspace, "local", WorkspaceConstants.LOGGING_CONFIG) + if not os.path.isfile(log_config_file_path): + log_config_file_path = os.path.join(os.path.dirname(__file__), WorkspaceConstants.LOGGING_CONFIG) with open(log_config_file_path, "r") as f: dict_config = json.load(f) - self.args.log_config = None self.args.config_folder = "config" self.args.job_id = SimulatorConstants.JOB_NAME self.args.client_config = os.path.join(self.args.config_folder, JobConstants.CLIENT_JOB_CONFIG) @@ -669,9 +689,12 @@ def _pick_next_client(self): def do_one_task(self, client, num_of_threads, gpu, lock, timeout=60.0, task_name=RunnerTask.TASK_EXEC): open_port = get_open_ports(1)[0] client_workspace = os.path.join(self.args.workspace, client.client_name) - logging_config = os.path.join( - self.args.workspace, client.client_name, "local", WorkspaceConstants.LOGGING_CONFIG - ) + if self.args.log_config: + logging_config = self.args.log_config + else: + logging_config = os.path.join( + self.args.workspace, client.client_name, "local", WorkspaceConstants.LOGGING_CONFIG + ) decomposer_module = ConfigService.get_str_var( name=ConfigVarName.DECOMPOSER_MODULE, conf=SystemConfigs.RESOURCES_CONF ) diff --git a/nvflare/private/fed/client/admin_commands.py b/nvflare/private/fed/client/admin_commands.py index ed4488c473..67797ac0c5 100644 --- a/nvflare/private/fed/client/admin_commands.py +++ b/nvflare/private/fed/client/admin_commands.py @@ -246,6 +246,32 @@ def process(self, data: Shareable, fl_ctx: FLContext): return None +class ConfigureJobLogCommand(CommandProcessor): + """To implement the configure_job_log command.""" + + def get_command_name(self) -> str: + """To get the command name. + + Returns: AdminCommandNames.CONFIGURE_JOB_LOG + + """ + return AdminCommandNames.CONFIGURE_JOB_LOG + + def process(self, data: Shareable, fl_ctx: FLContext): + """Called to process the configure_job_log command. + + Args: + data: process data + fl_ctx: FLContext + + Returns: configure_job_log command message + + """ + client_runner = fl_ctx.get_prop(FLContextKey.RUNNER) + if client_runner: + client_runner.configure_job_log(data, fl_ctx) + + class AdminCommands(object): """AdminCommands contains all the commands for processing the commands from the parent process.""" @@ -257,6 +283,7 @@ class AdminCommands(object): ShowStatsCommand(), ShowErrorsCommand(), ResetErrorsCommand(), + ConfigureJobLogCommand(), ] @staticmethod diff --git a/nvflare/private/fed/client/client_engine.py b/nvflare/private/fed/client/client_engine.py index 4799ec91a5..034d8b8a50 100644 --- a/nvflare/private/fed/client/client_engine.py +++ b/nvflare/private/fed/client/client_engine.py @@ -464,6 +464,9 @@ def get_current_run_info(self, job_id) -> ClientRunInfo: def get_errors(self, job_id): return self.client_executor.get_errors(job_id) + def configure_job_log(self, job_id, config): + self.client_executor.configure_job_log(job_id, config) + def reset_errors(self, job_id): self.client_executor.reset_errors(job_id) diff --git a/nvflare/private/fed/client/client_executor.py b/nvflare/private/fed/client/client_executor.py index 246e456cb2..2057f7f601 100644 --- a/nvflare/private/fed/client/client_executor.py +++ b/nvflare/private/fed/client/client_executor.py @@ -277,6 +277,27 @@ def get_errors(self, job_id): secure_log_traceback() return None + def configure_job_log(self, job_id, config): + """Configure the job log. + + Args: + job_id: the job_id + config: log config + + """ + try: + request = new_cell_message({}, config) + self.client.cell.fire_and_forget( + targets=self._job_fqcn(job_id), + channel=CellChannel.CLIENT_COMMAND, + topic=AdminCommandNames.CONFIGURE_JOB_LOG, + message=request, + optional=True, + ) + except Exception as e: + self.logger.error(f"configure_job_log execution exception: {secure_format_exception(e)}.") + secure_log_traceback() + def reset_errors(self, job_id): """Resets the error information. diff --git a/nvflare/private/fed/client/client_req_processors.py b/nvflare/private/fed/client/client_req_processors.py index a1aa221beb..2689bce4a5 100644 --- a/nvflare/private/fed/client/client_req_processors.py +++ b/nvflare/private/fed/client/client_req_processors.py @@ -15,11 +15,12 @@ from .info_coll_cmd import ClientInfoProcessor from .scheduler_cmds import CancelResourceProcessor, CheckResourceProcessor, ReportResourcesProcessor, StartJobProcessor from .shell_cmd import ShellCommandProcessor -from .sys_cmd import ReportEnvProcessor, SysInfoProcessor +from .sys_cmd import ConfigureSiteLogProcessor, ReportEnvProcessor, SysInfoProcessor from .training_cmds import ( # StartClientMGpuProcessor,; SetRunNumberProcessor, AbortAppProcessor, AbortTaskProcessor, ClientStatusProcessor, + ConfigureJobLogProcessor, DeleteRunNumberProcessor, DeployProcessor, NotifyJobStatusProcessor, @@ -52,6 +53,8 @@ class ClientRequestProcessors: ReportResourcesProcessor(), ReportEnvProcessor(), NotifyJobStatusProcessor(), + ConfigureJobLogProcessor(), + ConfigureSiteLogProcessor(), ] @staticmethod diff --git a/nvflare/private/fed/client/client_runner.py b/nvflare/private/fed/client/client_runner.py index e258b4caf8..33422e188a 100644 --- a/nvflare/private/fed/client/client_runner.py +++ b/nvflare/private/fed/client/client_runner.py @@ -35,6 +35,7 @@ from nvflare.apis.utils.reliable_message import ReliableMessage from nvflare.apis.utils.task_utils import apply_filters from nvflare.fuel.f3.cellnet.fqcn import FQCN +from nvflare.fuel.utils.log_utils import dynamic_log_config from nvflare.private.defs import SpecialTaskName, TaskConstant from nvflare.private.fed.client.client_engine_executor_spec import ClientEngineExecutorSpec, TaskAssignment from nvflare.private.fed.tbi import TBI @@ -719,3 +720,9 @@ def _handle_do_task(self, topic: str, request: Shareable, fl_ctx: FLContext) -> task = TaskAssignment(name=task_name, task_id=task_id, data=request) reply = self._process_task(task, fl_ctx) return reply + + def configure_job_log(self, request: Shareable, fl_ctx: FLContext) -> Shareable: + dynamic_log_config(request, self.engine.get_workspace(), self.job_id) + self.log_info(fl_ctx, f"configured job {self.job_id} server log") + + return make_reply(ReturnCode.OK) diff --git a/nvflare/private/fed/client/sys_cmd.py b/nvflare/private/fed/client/sys_cmd.py index 1bf20fd0c6..43a5645bc7 100644 --- a/nvflare/private/fed/client/sys_cmd.py +++ b/nvflare/private/fed/client/sys_cmd.py @@ -24,7 +24,8 @@ from nvflare.apis.fl_constant import FLContextKey, SystemComponents from nvflare.apis.fl_context import FLContext -from nvflare.private.admin_defs import Message +from nvflare.fuel.utils.log_utils import dynamic_log_config +from nvflare.private.admin_defs import Message, ok_reply from nvflare.private.defs import SysCommandTopic from nvflare.private.fed.client.admin import RequestProcessor @@ -80,3 +81,17 @@ def process(self, req: Message, app_ctx) -> Message: } message = Message(topic="reply_" + req.topic, body=json.dumps(env)) return message + + +class ConfigureSiteLogProcessor(RequestProcessor): + def get_topics(self) -> List[str]: + return [SysCommandTopic.CONFIGURE_SITE_LOG] + + def process(self, req: Message, app_ctx) -> Message: + engine = app_ctx + fl_ctx = engine.new_context() + workspace = fl_ctx.get_prop(FLContextKey.WORKSPACE_OBJECT) + + dynamic_log_config(req.body, workspace) + + return ok_reply(topic=f"reply_{req.topic}", body="OK") diff --git a/nvflare/private/fed/client/training_cmds.py b/nvflare/private/fed/client/training_cmds.py index e554568fa4..759750112c 100644 --- a/nvflare/private/fed/client/training_cmds.py +++ b/nvflare/private/fed/client/training_cmds.py @@ -153,6 +153,21 @@ def process(self, req: Message, app_ctx) -> Message: return ok_reply(topic=f"reply_{req.topic}", body=result) +class ConfigureJobLogProcessor(RequestProcessor): + def get_topics(self) -> List[str]: + return [TrainingTopic.CONFIGURE_JOB_LOG] + + def process(self, req: Message, app_ctx) -> Message: + engine = app_ctx + if not isinstance(engine, ClientEngineInternalSpec): + raise TypeError("engine must be ClientEngineInternalSpec, but got {}".format(type(engine))) + + job_id = req.get_header(RequestHeader.JOB_ID) + engine.configure_job_log(job_id, req.body) + + return ok_reply(topic=f"reply_{req.topic}", body="") + + class ClientStatusProcessor(RequestProcessor): def get_topics(self) -> List[str]: return [TrainingTopic.CHECK_STATUS] diff --git a/nvflare/private/fed/server/job_cmds.py b/nvflare/private/fed/server/job_cmds.py index 74d55ed10a..8cb5eba012 100644 --- a/nvflare/private/fed/server/job_cmds.py +++ b/nvflare/private/fed/server/job_cmds.py @@ -92,6 +92,13 @@ def get_spec(self): enabled=False, confirm=ConfirmMethod.AUTH, ), + CommandSpec( + name=AdminCommandNames.CONFIGURE_JOB_LOG, + description="configure logging of a job", + usage=f"{AdminCommandNames.CONFIGURE_JOB_LOG} job_id server|client ... config", + handler_func=self.configure_job_log, + authz_func=self.authorize_configure_job_log, + ), CommandSpec( name=AdminCommandNames.START_APP, description="start the FL app", @@ -230,6 +237,12 @@ def authorize_job(self, conn: Connection, args: List[str]): return PreAuthzReturnCode.REQUIRE_AUTHZ + def authorize_configure_job_log(self, conn: Connection, args: List[str]): + if len(args) < 4: + conn.append_error("syntax error: please provide job_id, target_type, and config") + return PreAuthzReturnCode.ERROR + self.authorize_job(conn, args[:-1]) + def _start_app_on_clients(self, conn: Connection, job_id: str) -> bool: engine = conn.app_ctx client_names = conn.get_prop(self.TARGET_CLIENT_NAMES, None) @@ -321,6 +334,39 @@ def delete_job_id(self, conn: Connection, args: List[str]): conn.append_success("") + def configure_job_log(self, conn: Connection, args: List[str]): + if len(args) < 4: + conn.append_error("syntax error: please provide job_id, target_type, and config") + return + + job_id = args[1] + target_type = args[2] + config = args[-1] + + engine = conn.app_ctx + if not isinstance(engine, ServerEngine): + raise TypeError("engine must be ServerEngine but got {}".format(type(engine))) + + if target_type in [self.TARGET_TYPE_SERVER, self.TARGET_TYPE_ALL]: + engine.configure_job_log(str(job_id), config) + conn.append_string(f"successfully configured server job {job_id} log") + + if target_type in [self.TARGET_TYPE_CLIENT, self.TARGET_TYPE_ALL]: + message = new_message(conn, topic=TrainingTopic.CONFIGURE_JOB_LOG, body=config, require_authz=False) + message.set_header(RequestHeader.JOB_ID, str(job_id)) + replies = self.send_request_to_clients(conn, message) + self.process_replies_to_table(conn, replies) + conn.append_string( + f"successfully configured client job {job_id} logs of {conn.get_prop(self.TARGET_CLIENT_NAMES)}" + ) + + if target_type not in [self.TARGET_TYPE_ALL, self.TARGET_TYPE_CLIENT, self.TARGET_TYPE_SERVER]: + conn.append_string( + "invalid target type {}. Usage: configure_job_log job_id server|client ... config".format( + target_type + ) + ) + def list_jobs(self, conn: Connection, args: List[str]): try: parser = _create_list_job_cmd_parser() diff --git a/nvflare/private/fed/server/server_commands.py b/nvflare/private/fed/server/server_commands.py index cf1361971c..372ec82bdf 100644 --- a/nvflare/private/fed/server/server_commands.py +++ b/nvflare/private/fed/server/server_commands.py @@ -423,6 +423,31 @@ def process(self, data: Shareable, fl_ctx: FLContext): return "Success" +class ConfigureJobLogCommand(CommandProcessor): + """To implement the configure_job_log command.""" + + def get_command_name(self) -> str: + """To get the command name. + + Returns: ServerCommandNames.CONFIGURE_JOB_LOG + + """ + return ServerCommandNames.CONFIGURE_JOB_LOG + + def process(self, data: Shareable, fl_ctx: FLContext): + """Called to process the configure_job_log command. + + Args: + data: process data + fl_ctx: FLContext + + """ + server_runner = fl_ctx.get_prop(FLContextKey.RUNNER) + + if server_runner: + server_runner.configure_job_log(data, fl_ctx) + + class AppCommandProcessor(CommandProcessor): def get_command_name(self) -> str: """To get the command name. @@ -480,6 +505,7 @@ class ServerCommands(object): ResetErrorsCommand(), HeartbeatCommand(), ServerStateCommand(), + ConfigureJobLogCommand(), AppCommandProcessor(), ] diff --git a/nvflare/private/fed/server/server_engine.py b/nvflare/private/fed/server/server_engine.py index f21d836322..5b96a99471 100644 --- a/nvflare/private/fed/server/server_engine.py +++ b/nvflare/private/fed/server/server_engine.py @@ -861,6 +861,18 @@ def reset_errors(self, job_id) -> str: return f"reset the server error stats for job: {job_id}" + def configure_job_log(self, job_id, data) -> dict: + try: + self.send_command_to_child_runner_process( + job_id=job_id, + command_name=ServerCommandNames.CONFIGURE_JOB_LOG, + command_data=data, + ) + except Exception as ex: + self.logger.error(f"Failed to configure_job_log for JOB: {job_id}: {secure_format_exception(ex)}") + + return f"configured log for job: {job_id}" + def _send_admin_requests(self, requests, fl_ctx: FLContext, timeout_secs=10) -> List[ClientReply]: return self.server.admin_server.send_requests(requests, fl_ctx, timeout_secs=timeout_secs) diff --git a/nvflare/private/fed/server/server_runner.py b/nvflare/private/fed/server/server_runner.py index 81c47f4848..97a73dfa81 100644 --- a/nvflare/private/fed/server/server_runner.py +++ b/nvflare/private/fed/server/server_runner.py @@ -26,6 +26,7 @@ from nvflare.apis.utils.fl_context_utils import add_job_audit_event from nvflare.apis.utils.reliable_message import ReliableMessage from nvflare.apis.utils.task_utils import apply_filters +from nvflare.fuel.utils.log_utils import dynamic_log_config from nvflare.private.defs import SpecialTaskName, TaskConstant from nvflare.private.fed.tbi import TBI from nvflare.private.privacy_manager import Scope @@ -554,3 +555,9 @@ def get_persist_state(self, fl_ctx: FLContext) -> dict: def restore(self, state_data: dict, fl_ctx: FLContext): self.job_id = state_data.get("job_id") self.current_wf_index = int(state_data.get("current_wf_index", 0)) + + def configure_job_log(self, data, fl_ctx: FLContext) -> Shareable: + dynamic_log_config(data, self.engine.get_workspace(), self.job_id) + self.log_info(fl_ctx, f"configured job {self.job_id} server log") + + return make_reply(ReturnCode.OK) diff --git a/nvflare/private/fed/server/sys_cmd.py b/nvflare/private/fed/server/sys_cmd.py index 6fb89266ae..3313411e12 100644 --- a/nvflare/private/fed/server/sys_cmd.py +++ b/nvflare/private/fed/server/sys_cmd.py @@ -20,10 +20,13 @@ from nvflare.fuel.hci.conn import Connection from nvflare.fuel.hci.proto import MetaKey from nvflare.fuel.hci.reg import CommandModule, CommandModuleSpec, CommandSpec +from nvflare.fuel.hci.server.authz import PreAuthzReturnCode +from nvflare.fuel.utils.log_utils import dynamic_log_config from nvflare.private.admin_defs import MsgHeader, ReturnCode from nvflare.private.defs import SysCommandTopic from nvflare.private.fed.server.admin import new_message from nvflare.private.fed.server.cmd_utils import CommandUtil +from nvflare.private.fed.server.server_engine import ServerEngine from nvflare.security.logging import secure_format_exception @@ -60,6 +63,14 @@ def get_spec(self): authz_func=self.authorize_server_operation, visible=True, ), + CommandSpec( + name="configure_site_log", + description="configure logging of a site", + usage="configure_site_log server|client ... config", + handler_func=self.configure_site_log, + authz_func=self.authorize_configure_site_log, + visible=True, + ), CommandSpec( name="report_resources", description="get the resources info", @@ -87,6 +98,12 @@ def get_spec(self): ], ) + def authorize_configure_site_log(self, conn: Connection, args: List[str]): + if len(args) < 3: + conn.append_error("syntax error: please provide target_type and config") + return PreAuthzReturnCode.ERROR + self.authorize_server_operation(conn, args[:-1]) + def sys_info(self, conn: Connection, args: [str]): if len(args) < 2: conn.append_error("syntax error: missing site names") @@ -116,6 +133,36 @@ def sys_info(self, conn: Connection, args: [str]): conn.append_string("invalid target type {}. Usage: sys_info server|client ".format(target_type)) + def configure_site_log(self, conn: Connection, args: [str]): + if len(args) < 3: + conn.append_error("syntax error: please provide target_type and config") + return + + target_type = args[1] + config = args[-1] + + if target_type in [self.TARGET_TYPE_SERVER, self.TARGET_TYPE_ALL]: + engine = conn.app_ctx + if not isinstance(engine, ServerEngine): + raise TypeError("engine must be ServerEngine but got {}".format(type(engine))) + + workspace = engine.get_workspace() + dynamic_log_config(config, workspace) + conn.append_string("successfully configured server site log") + + if target_type in [self.TARGET_TYPE_CLIENT, self.TARGET_TYPE_ALL]: + message = new_message(conn, topic=SysCommandTopic.CONFIGURE_SITE_LOG, body=config, require_authz=True) + replies = self.send_request_to_clients(conn, message) + self.process_replies_to_table(conn, replies) + conn.append_string(f"successfully configured client site logs of {conn.get_prop(self.TARGET_CLIENT_NAMES)}") + + if target_type not in [self.TARGET_TYPE_ALL, self.TARGET_TYPE_CLIENT, self.TARGET_TYPE_SERVER]: + conn.append_string( + "invalid target type {}. Usage: configure_site_log server|client ... config".format( + target_type + ) + ) + def _process_replies(self, conn, replies): if not replies: conn.append_error("no responses from clients") diff --git a/nvflare/security/security.py b/nvflare/security/security.py index 903a053f40..05009d9f24 100644 --- a/nvflare/security/security.py +++ b/nvflare/security/security.py @@ -33,6 +33,7 @@ class CommandCategory(object): AC.START_APP: CommandCategory.MANAGE_JOB, AC.DELETE_JOB: CommandCategory.MANAGE_JOB, AC.DELETE_WORKSPACE: CommandCategory.MANAGE_JOB, + AC.CONFIGURE_JOB_LOG: CommandCategory.MANAGE_JOB, AC.CHECK_STATUS: CommandCategory.VIEW, AC.SHOW_SCOPES: CommandCategory.VIEW, AC.SHOW_STATS: CommandCategory.VIEW, @@ -48,6 +49,7 @@ class CommandCategory(object): AC.REMOVE_CLIENT: CommandCategory.OPERATE, AC.SET_TIMEOUT: CommandCategory.OPERATE, AC.CALL: CommandCategory.OPERATE, + AC.CONFIGURE_SITE_LOG: CommandCategory.OPERATE, AC.SHELL_CAT: CommandCategory.SHELL_COMMANDS, AC.SHELL_GREP: CommandCategory.SHELL_COMMANDS, AC.SHELL_HEAD: CommandCategory.SHELL_COMMANDS,