dynamic logging admin command

docs/resources/log.config

@@ -17,8 +17,8 @@
     },
     "filters": {
         "FLFilter": {
-            "()": "nvflare.fuel.utils.log_utils.FLFilter",
-            "logger_names": ["nvflare.app_common", "nvflare.app_opt"]
+            "()": "nvflare.fuel.utils.log_utils.LoggerNameFilter",
+            "logger_names": ["custom", "nvflare.app_common", "nvflare.app_opt"]
         }
     },
     "handlers": {

nvflare/apis/fl_constant.py

@@ -245,6 +245,8 @@ class AdminCommandNames(object):
     SHELL_TAIL = "tail"
     SHELL_GREP = "grep"
     APP_COMMAND = "app_command"
+    CONFIGURE_JOB_LOG = "configure_job_log"
+    CONFIGURE_SITE_LOG = "configure_site_log"
 
 
 class ServerCommandNames(object):
@@ -263,6 +265,7 @@ class ServerCommandNames(object):
     HANDLE_DEAD_JOB = "handle_dead_job"
     SERVER_STATE = "server_state"
     APP_COMMAND = "app_command"
+    CONFIGURE_JOB_LOG = "configure_job_log"
 
 
 class ServerCommandKey(object):

nvflare/fuel/utils/log_utils.py

@@ -10,7 +10,7 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License.s
 import inspect
 import json
 import logging
@@ -20,32 +20,59 @@
 from logging import Logger
 from logging.handlers import RotatingFileHandler
 
+from nvflare.apis.fl_constant import WorkspaceConstants
 from nvflare.apis.workspace import Workspace
 
 
 class ANSIColor:
-    GREY = "38"
-    YELLOW = "33"
-    RED = "31"
-    BOLD_RED = "31;1"
-    CYAN = "36"
-    RESET = "0"
-
+    # Basic ANSI color codes
+    COLORS = {
+        "black": "30",
+        "red": "31",
+        "bold_red": "31;1",
+        "green": "32",
+        "yellow": "33",
+        "blue": "34",
+        "magenta": "35",
+        "cyan": "36",
+        "white": "37",
+        "grey": "38",
+        "reset": "0",
+    }
+
+    # Default logger level:color mappings
+    DEFAULT_LEVEL_COLORS = {
+        "NOTSET": COLORS["grey"],
+        "DEBUG": COLORS["grey"],
+        "INFO": COLORS["grey"],
+        "WARNING": COLORS["yellow"],
+        "ERROR": COLORS["red"],
+        "CRITICAL": COLORS["bold_red"],
+    }
+
+    @classmethod
+    def colorize(cls, text: str, color: str) -> str:
+        """Wrap text with the given ANSI SGR color.
+
+                Args:
+                    text (str): text to colorize.
+                    color (str): ANSI SGR color code or color name defined in ANSIColor.COLORS.
+
+                Returns:
+                    colorized text
+        s
+        """
+        if not any(c.isdigit() for c in color):
+            color = cls.COLORS.get(color.lower(), cls.COLORS["reset"])
 
-DEFAULT_LEVEL_COLORS = {
-    "DEBUG": ANSIColor.GREY,
-    "INFO": ANSIColor.GREY,
-    "WARNING": ANSIColor.YELLOW,
-    "ERROR": ANSIColor.RED,
-    "CRITICAL": ANSIColor.BOLD_RED,
-}
+        return f"\x1b[{color}m{text}\x1b[{cls.COLORS['reset']}m"
 
 
 class BaseFormatter(logging.Formatter):
     def __init__(self, fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt=None, style="%"):
-        """BaseFormatter is the default formatter for log records.
+        """Default formatter for log records.
 
-        Shortens logger %(name)s to the suffix. Full name can be accessed with %(fullName)s
+        Shortens logger %(name)s to the basenames. Full name can be accessed with %(fullName)s
 
         Args:
             fmt (str): format string which uses LogRecord attributes.
@@ -70,65 +97,59 @@ def __init__(
         fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
         datefmt=None,
         style="%",
-        level_colors=DEFAULT_LEVEL_COLORS,
-        logger_names=None,
-        logger_color=ANSIColor.CYAN,
+        level_colors=ANSIColor.DEFAULT_LEVEL_COLORS,
+        logger_colors={},
     ):
-        """ColorFormatter to format colors based on log levels. Optionally can color logger_names.
+        """Format colors based on log levels. Optionally can provide mapping based on logger namess.
 
         Args:
             fmt (str): format string which uses LogRecord attributes.
             datefmt (str): date/time format string. Defaults to '%Y-%m-%d %H:%M:%S'.
             style (str): style character '%' '{' or '$' for format string.
-            level_colors (Dict[str, str]): dict of levelname: ANSI color. Defaults to
-                {
-                    "DEBUG": ANSIColor.GREY,
-                    "INFO": ANSIColor.GREY,
-                    "WARNING": ANSIColor.YELLOW,
-                    "ERROR": ANSIColor.RED,
-                    "CRITICAL": ANSIColor.BOLD_RED,
-                }
-            logger_names (List[str]): list of logger names to apply logger_color.
-            logger_color (int): ANSI custom color for logger_names.
+            level_colors (Dict[str, str]): dict of levelname: ANSI color. Defaults to ANSIColor.DEFAULT_LEVEL_COLORS.
+            logger_colors (Dict[str, str]): dict of logger_name: ANSI colors. Defaults to {}.
 
         """
         super().__init__(fmt=fmt, datefmt=datefmt, style=style)
-        self.logger_names = logger_names or []
-        self.logger_color = logger_color
         self.level_colors = level_colors
+        self.logger_colors = logger_colors
 
     def format(self, record):
         super().format(record)
 
-        if record.levelno <= logging.INFO and any(
-            record.name.startswith(logger_name) for logger_name in self.logger_names
-        ):
-            # Apply logger_color to logger_names
-            log_fmt = ansi_sgr(self.logger_color) + self.fmt + ansi_sgr(ANSIColor.RESET)
-        else:
-            # Apply level_colors based on record levelname
-            log_fmt = (
-                ansi_sgr(self.level_colors.get(record.levelname, ANSIColor.GREY)) + self.fmt + ansi_sgr(ANSIColor.RESET)
-            )
+        # Apply level_colors based on record levelname
+        log_color = self.level_colors.get(record.levelname, "reset")
+
+        # Apply logger_color to logger_names if INFO or below
+        if record.levelno <= logging.INFO:
+            log_color = self.logger_colors.get(record.name, log_color)
+
+        log_fmt = ANSIColor.colorize(self.fmt, log_color)
 
         formatter = logging.Formatter(log_fmt)
         return formatter.format(record)
 
 
 class JsonFormatter(BaseFormatter):
     def __init__(
-        self, fmt="%(asctime)s - %(name)s - %(fullName)s - %(levelname)s - %(message)s", datefmt=None, style="%"
+        self,
+        fmt="%(asctime)s - %(name)s - %(fullName)s - %(levelname)s - %(message)s",
+        datefmt=None,
+        style="%",
+        extract_brackets=True,
     ):
         """Format log records into JSON.
 
         Args:
             fmt (str): format string which uses LogRecord attributes. Attributes are used for JSON keys.
             datefmt (str): date/time format string. Defaults to '%Y-%m-%d %H:%M:%S'.
             style (str): style character '%' '{' or '$' for format string.
+            extract_bracket_fields (bool): whether to extract bracket fields of message into sub-dictionary. Defaults to True.
 
         """
         super().__init__(fmt=fmt, datefmt=datefmt, style=style)
         self.fmt_dict = self.generate_fmt_dict(self.fmt)
+        self.extract_brackets = extract_brackets
 
     def generate_fmt_dict(self, fmt: str) -> dict:
         # Parse the `fmt` string and create a mapping of keys to LogRecord attributes
@@ -144,7 +165,7 @@ def generate_fmt_dict(self, fmt: str) -> dict:
         return fmt_dict
 
     def extract_bracket_fields(self, message: str) -> dict:
-        # Extract bracketed fl_ctx_fields eg. [k1=v1, k2=v2...] into sub dict
+        # Extract bracketed fl_ctx_fields eg. [k1=v1, k2=v2...] into sub-dictionary
         bracket_fields = {}
         match = re.search(r"\[(.*?)\]:", message)
         if match:
@@ -162,7 +183,7 @@ def format(self, record) -> str:
         super().format(record)
 
         record.message = record.getMessage()
-        bracket_fields = self.extract_bracket_fields(record.message)
+        bracket_fields = self.extract_bracket_fields(record.message) if self.extract_brackets else None
         record.asctime = self.formatTime(record)
 
         formatted_message_dict = self.formatMessage(record)
@@ -177,8 +198,8 @@ def format(self, record) -> str:
         return json.dumps(message_dict, default=str)
 
 
-class FLFilter(logging.Filter):
-    def __init__(self, logger_names=["nvflare.app_common", "nvflare.app_opt"]):
+class LoggerNameFilter(logging.Filter):
+    def __init__(self, logger_names=["nvflare"]):
         """Filter log records based on logger names.
 
         Args:
@@ -189,15 +210,8 @@ def __init__(self, logger_names=["nvflare.app_common", "nvflare.app_opt"]):
         self.logger_names = logger_names
 
     def filter(self, record):
-        # Filter log records based on the logger name
-        fullName = record.fullName if hasattr(record, "fullName") else record.name
-        if any(fullName.startswith(name) for name in self.logger_names):
-            return record.levelno >= logging.INFO
-
-
-def ansi_sgr(code: str):
-    """ANSI Select Graphics Rendition."""
-    return "\x1b[" + code + "m"
+        name = record.fullName if hasattr(record, "fullName") else record.name
+        return any(name.startswith(logger_name) for logger_name in self.logger_names)
 
 
 def get_module_logger(module=None, name=None):
@@ -213,51 +227,81 @@ def get_obj_logger(obj):
 
 
 def get_script_logger():
+    # Get script logger name based on filename and package. If not in a package, default to custom.
     caller_frame = inspect.stack()[1]
     package = caller_frame.frame.f_globals.get("__package__", "")
     file = caller_frame.frame.f_globals.get("__file__", "")
 
     return logging.getLogger(
-        f"{package + '.' if package else ''}{os.path.splitext(os.path.basename(file))[0] if file else ''}"
+        f"{package if package else 'custom'}{'.' + os.path.splitext(os.path.basename(file))[0] if file else ''}"
     )
 
 
-def update_filenames(obj, dir_path: str = "", file_prefix: str = ""):
-    """Update 'filename' keys in JSON objects with dir_path and file_prefix."""
-    if "filename" in obj and isinstance(obj["filename"], str):
-        filename = obj["filename"]
-        if file_prefix:
-            filename = os.path.join(os.path.dirname(filename), file_prefix + "_" + os.path.basename(filename))
-        obj["filename"] = os.path.join(dir_path, filename)
-    return obj
+def configure_logging(workspace: Workspace, dir_path: str = "", file_prefix: str = ""):
+    # Read log_config.json from workspace, update with file_prefix, and apply to dir_path
+    log_config_file_path = workspace.get_log_config_file_path()
+    assert os.path.isfile(log_config_file_path), f"missing log config file {log_config_file_path}"
 
+    with open(log_config_file_path, "r") as f:
+        dict_config = json.load(f)
 
-def read_log_config(file, dir_path: str = "", file_prefix: str = "") -> dict:
-    """
-    Reads JSON logging configuration file and returns config dictionary.
-    Updates 'filename' keys with dir_path for dynamic locations.
+    apply_log_config(dict_config, dir_path, file_prefix)
 
-    Args:
-        file (str): Path to the configuration file.
-        dir_path (str): Update filename keys with dir_path.
 
-    Returns:
-        config (dict)
-    """
-    try:
-        with open(file, "r") as f:
-            config = json.load(f, object_hook=lambda obj: update_filenames(obj, dir_path, file_prefix))
-        return config
-    except Exception as e:
-        raise ValueError(f"Unrecognized logging configuration format. Failed to parse JSON: {e}.")
+def apply_log_config(dict_config, dir_path: str = "", file_prefix: str = ""):
+    # Update log config dictionary with file_prefix, and apply to dir_path
+    stack = [dict_config]
+    while stack:
+        current_dict = stack.pop()
+        for key, value in current_dict.items():
+            if isinstance(value, dict):
+                stack.append(value)
+            elif key == "filename":
+                if file_prefix:
+                    value = os.path.join(os.path.dirname(value), file_prefix + "_" + os.path.basename(value))
+                current_dict[key] = os.path.join(dir_path, value)
 
+    logging.config.dictConfig(dict_config)
 
-def configure_logging(workspace: Workspace, dir_path: str = "", file_prefix: str = ""):
-    log_config_file_path = workspace.get_log_config_file_path()
-    assert os.path.isfile(log_config_file_path), f"missing log config file {log_config_file_path}"
 
-    dict_config = read_log_config(log_config_file_path, dir_path, file_prefix)
-    logging.config.dictConfig(dict_config)
+def dynamic_log_config(config: str, workspace: Workspace, job_id: str = None):
+    # Dynamically configure log given a config (filepath, levelname, levelnumber, 'reload'), apply the config to the proper locations.
+    if not isinstance(config, str):
+        raise ValueError(
+            f"Unsupported config type. Expect config to be string filepath, levelname, levelnumber, or 'reload' but got {type(config)}"
+        )
+
+    if config == "reload":
+        config = workspace.get_log_config_file_path()
+
+    if os.path.isfile(config):
+        # Read confg file
+        with open(config, "r") as f:
+            dict_config = json.load(f)
+
+        if job_id:
+            dir_path = workspace.get_run_dir(job_id)
+        else:
+            dir_path = workspace.get_root_dir()
+
+            # overwrite log_config.json of site
+            with open(os.path.join(workspace.get_site_config_dir(), WorkspaceConstants.LOGGING_CONFIG), "w") as f:
+                f.write(json.dumps(dict_config))
+
+        apply_log_config(dict_config, dir_path)
+
+    else:
+        # Set level of root logger based on levelname or levelnumber
+        if config.isdigit():
+            level = int(config)
+            if not (0 <= level <= 50):
+                raise ValueError(f"Invalid logging level: {level}")
+        else:
+            level = getattr(logging, config.upper(), None)
+            if level is None:
+                raise ValueError(f"Invalid logging level: {config}")
+
+        logging.getLogger().setLevel(level)
 
 
 def add_log_file_handler(log_file_name):
@@ -269,7 +313,7 @@ def add_log_file_handler(log_file_name):
     root_logger.addHandler(file_handler)
 
 
-def print_logger_hierarchy(package_name="nvflare", level_colors=DEFAULT_LEVEL_COLORS):
+def print_logger_hierarchy(package_name="nvflare", level_colors=ANSIColor.DEFAULT_LEVEL_COLORS):
     all_loggers = logging.root.manager.loggerDict
 
     # Filter for package loggers based on package_name
@@ -300,8 +344,8 @@ def print_hierarchy(logger_name, indent_level=0):
         level_display = f"{level_name} (SET)" if not is_unset else level_name
 
         # Print the logger with color and indentation
-        color = level_colors.get(level_name, ANSIColor.RESET)
-        print("    " * indent_level + f"{ansi_sgr(color)}{logger_name} [{level_display}]{ansi_sgr(ANSIColor.RESET)}")
+        color = level_colors.get(level_name, ANSIColor.COLORS["reset"])
+        print("    " * indent_level + ANSIColor.colorize(f"{logger_name} [{level_display}]", color))
 
         # Find child loggers based on the current hierarchy level
         for name in sorted_package_loggers: