From 621c89411cd4f71101ad8247952d9c3f47f2a5b8 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Wed, 6 Jan 2021 09:53:11 -0500 Subject: [PATCH 01/22] FIX: Convert timing values to datetimes from strings * exclude nodes without timing information from Gantt chart * fall back on "id" or empty string if no "name" in node --- nipype/utils/draw_gantt_chart.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index 3ae4b77246..fbfe502afe 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -307,7 +307,7 @@ def draw_nodes(start, nodes_list, cores, minute_scale, space_between_minutes, co "offset": offset, "scale_duration": scale_duration, "color": color, - "node_name": node["name"], + "node_name": node.get("name", node.get("id", "")), "node_dur": node["duration"] / 60.0, "node_start": node_start.strftime("%Y-%m-%d %H:%M:%S"), "node_finish": node_finish.strftime("%Y-%m-%d %H:%M:%S"), @@ -527,6 +527,20 @@ def generate_gantt_chart( # Read in json-log to get list of node dicts nodes_list = log_to_dict(logfile) + # Only include nodes with timing information, and covert timestamps + # from strings to datetimes + nodes_list = [{ + k: datetime.datetime.strptime( + i[k], "%Y-%m-%dT%H:%M:%S.%f" + ) if k in {"start", "finish"} else i[k] for k in i + } for i in nodes_list if "start" in i and "finish" in i] + + for node in nodes_list: + if "duration" not in node: + node["duration"] = ( + node["finish"] - node["start"] + ).total_seconds() + # Create the header of the report with useful information start_node = nodes_list[0] last_node = nodes_list[-1] From 2cf2d37b83496a87f0d2c37a8af940e3023d60ee Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Wed, 6 Jan 2021 10:16:59 -0500 Subject: [PATCH 02/22] REF: Reduce double logging from exception to warning --- nipype/utils/draw_gantt_chart.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index fbfe502afe..8c003b98b6 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -10,6 +10,7 @@ import simplejson as json from collections import OrderedDict +from warnings import warn # Pandas try: @@ -66,9 +67,9 @@ def create_event_dict(start_time, nodes_list): finish_delta = (node["finish"] - start_time).total_seconds() # Populate dictionary - if events.get(start_delta) or events.get(finish_delta): + if events.get(start_delta): err_msg = "Event logged twice or events started at exact same time!" - raise KeyError(err_msg) + warn(str(KeyError(err_msg)), category=Warning) events[start_delta] = start_node events[finish_delta] = finish_node From 2e50f46be0adb71e77267025ca2a5076e1d49db5 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Wed, 6 Jan 2021 12:35:54 -0500 Subject: [PATCH 03/22] TST: Add test for `draw_gantt_chart` --- .../pipeline/plugins/tests/test_callback.py | 33 +++++++++++++++++++ nipype/utils/draw_gantt_chart.py | 5 +-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/nipype/pipeline/plugins/tests/test_callback.py b/nipype/pipeline/plugins/tests/test_callback.py index f7606708c7..34e7cff2ee 100644 --- a/nipype/pipeline/plugins/tests/test_callback.py +++ b/nipype/pipeline/plugins/tests/test_callback.py @@ -60,3 +60,36 @@ def test_callback_exception(tmpdir, plugin, stop_on_first_crash): sleep(0.5) # Wait for callback to be called (python 2.7) assert so.statuses == [("f_node", "start"), ("f_node", "exception")] + + +@pytest.mark.parametrize("plugin", ["Linear", "MultiProc", "LegacyMultiProc"]) +def test_callback_gantt(tmpdir, plugin): + import logging + import logging.handlers + + from os import path + + from nipype.utils.profiler import log_nodes_cb + from nipype.utils.draw_gantt_chart import generate_gantt_chart + + log_filename = 'callback.log' + logger = logging.getLogger('callback') + logger.setLevel(logging.DEBUG) + handler = logging.FileHandler(log_filename) + logger.addHandler(handler) + + #create workflow + wf = pe.Workflow(name="test", base_dir=tmpdir.strpath) + f_node = pe.Node( + niu.Function(function=func, input_names=[], output_names=[]), name="f_node" + ) + wf.add_nodes([f_node]) + wf.config["execution"] = {"crashdump_dir": wf.base_dir, "poll_sleep_duration": 2} + + plugin_args = {"status_callback": log_nodes_cb} + if plugin != "Linear": + plugin_args['n_procs'] = 8 + wf.run(plugin=plugin, plugin_args=plugin_args) + + generate_gantt_chart('callback.log', 1 if plugin == "Linear" else 8) + assert path.exists('callback.log.html') \ No newline at end of file diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index 8c003b98b6..c373ba24fe 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -153,13 +153,14 @@ def calculate_resource_timeseries(events, resource): all_res = 0.0 # Iterate through the events + nan = {"Unknown", "N/A"} for _, event in sorted(events.items()): if event["event"] == "start": - if resource in event and event[resource] != "Unknown": + if resource in event and event[resource] not in nan: all_res += float(event[resource]) current_time = event["start"] elif event["event"] == "finish": - if resource in event and event[resource] != "Unknown": + if resource in event and event[resource] not in nan: all_res -= float(event[resource]) current_time = event["finish"] res[current_time] = all_res From 7272623cf9d0c371f0738a24054216b4b0fd69ff Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 8 Jan 2021 09:35:29 -0500 Subject: [PATCH 04/22] STY: Automatic linting by pre-commit --- .../pipeline/plugins/tests/test_callback.py | 12 ++++++------ nipype/utils/draw_gantt_chart.py | 19 +++++++++++-------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/nipype/pipeline/plugins/tests/test_callback.py b/nipype/pipeline/plugins/tests/test_callback.py index 34e7cff2ee..e568a2bd72 100644 --- a/nipype/pipeline/plugins/tests/test_callback.py +++ b/nipype/pipeline/plugins/tests/test_callback.py @@ -72,13 +72,13 @@ def test_callback_gantt(tmpdir, plugin): from nipype.utils.profiler import log_nodes_cb from nipype.utils.draw_gantt_chart import generate_gantt_chart - log_filename = 'callback.log' - logger = logging.getLogger('callback') + log_filename = "callback.log" + logger = logging.getLogger("callback") logger.setLevel(logging.DEBUG) handler = logging.FileHandler(log_filename) logger.addHandler(handler) - #create workflow + # create workflow wf = pe.Workflow(name="test", base_dir=tmpdir.strpath) f_node = pe.Node( niu.Function(function=func, input_names=[], output_names=[]), name="f_node" @@ -88,8 +88,8 @@ def test_callback_gantt(tmpdir, plugin): plugin_args = {"status_callback": log_nodes_cb} if plugin != "Linear": - plugin_args['n_procs'] = 8 + plugin_args["n_procs"] = 8 wf.run(plugin=plugin, plugin_args=plugin_args) - generate_gantt_chart('callback.log', 1 if plugin == "Linear" else 8) - assert path.exists('callback.log.html') \ No newline at end of file + generate_gantt_chart("callback.log", 1 if plugin == "Linear" else 8) + assert path.exists("callback.log.html") diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index c373ba24fe..aed861f7ad 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -531,17 +531,20 @@ def generate_gantt_chart( # Only include nodes with timing information, and covert timestamps # from strings to datetimes - nodes_list = [{ - k: datetime.datetime.strptime( - i[k], "%Y-%m-%dT%H:%M:%S.%f" - ) if k in {"start", "finish"} else i[k] for k in i - } for i in nodes_list if "start" in i and "finish" in i] + nodes_list = [ + { + k: datetime.datetime.strptime(i[k], "%Y-%m-%dT%H:%M:%S.%f") + if k in {"start", "finish"} + else i[k] + for k in i + } + for i in nodes_list + if "start" in i and "finish" in i + ] for node in nodes_list: if "duration" not in node: - node["duration"] = ( - node["finish"] - node["start"] - ).total_seconds() + node["duration"] = (node["finish"] - node["start"]).total_seconds() # Create the header of the report with useful information start_node = nodes_list[0] From ea4def1837005a2a5546a4769d74105164bcbf47 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 8 Jan 2021 15:11:11 -0500 Subject: [PATCH 05/22] TST: Use tmpdir for Gantt test --- nipype/pipeline/plugins/tests/test_callback.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nipype/pipeline/plugins/tests/test_callback.py b/nipype/pipeline/plugins/tests/test_callback.py index e568a2bd72..c19687958a 100644 --- a/nipype/pipeline/plugins/tests/test_callback.py +++ b/nipype/pipeline/plugins/tests/test_callback.py @@ -72,7 +72,7 @@ def test_callback_gantt(tmpdir, plugin): from nipype.utils.profiler import log_nodes_cb from nipype.utils.draw_gantt_chart import generate_gantt_chart - log_filename = "callback.log" + log_filename = path.join(tmpdir, "callback.log") logger = logging.getLogger("callback") logger.setLevel(logging.DEBUG) handler = logging.FileHandler(log_filename) @@ -91,5 +91,7 @@ def test_callback_gantt(tmpdir, plugin): plugin_args["n_procs"] = 8 wf.run(plugin=plugin, plugin_args=plugin_args) - generate_gantt_chart("callback.log", 1 if plugin == "Linear" else 8) - assert path.exists("callback.log.html") + generate_gantt_chart( + path.join(tmpdir, "callback.log"), 1 if plugin == "Linear" else 8 + ) + assert path.exists(path.join(tmpdir, "callback.log.html")) From 169c09e6fe89e430d919756d755c10ead10c410d Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Fri, 8 Jan 2021 15:24:12 -0500 Subject: [PATCH 06/22] REF: Don't restrict nan timestamps to predetermined options --- nipype/utils/draw_gantt_chart.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index aed861f7ad..d94d339509 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -153,15 +153,20 @@ def calculate_resource_timeseries(events, resource): all_res = 0.0 # Iterate through the events - nan = {"Unknown", "N/A"} for _, event in sorted(events.items()): if event["event"] == "start": - if resource in event and event[resource] not in nan: - all_res += float(event[resource]) + if resource in event: + try: + all_res += float(event[resource]) + except ValueError: + next current_time = event["start"] elif event["event"] == "finish": - if resource in event and event[resource] not in nan: - all_res -= float(event[resource]) + if resource in event: + try: + all_res -= float(event[resource]) + except ValueError: + next current_time = event["finish"] res[current_time] = all_res From 9637b0f8139a9732ad38c75c554d46a678c3a823 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Thu, 1 Apr 2021 16:03:50 +0000 Subject: [PATCH 07/22] STY: Simplify warning Co-authored-by: Mathias Goncalves --- nipype/utils/draw_gantt_chart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index d94d339509..c7a1a5153f 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -69,7 +69,7 @@ def create_event_dict(start_time, nodes_list): # Populate dictionary if events.get(start_delta): err_msg = "Event logged twice or events started at exact same time!" - warn(str(KeyError(err_msg)), category=Warning) + warn(err_msg, category=Warning) events[start_delta] = start_node events[finish_delta] = finish_node From f336c22b7d21e9347e9941417d1e29aec682ef95 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Thu, 1 Apr 2021 16:04:25 +0000 Subject: [PATCH 08/22] REF: Remove unnecessary import Co-authored-by: Mathias Goncalves --- nipype/pipeline/plugins/tests/test_callback.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nipype/pipeline/plugins/tests/test_callback.py b/nipype/pipeline/plugins/tests/test_callback.py index c19687958a..02234522fa 100644 --- a/nipype/pipeline/plugins/tests/test_callback.py +++ b/nipype/pipeline/plugins/tests/test_callback.py @@ -65,7 +65,6 @@ def test_callback_exception(tmpdir, plugin, stop_on_first_crash): @pytest.mark.parametrize("plugin", ["Linear", "MultiProc", "LegacyMultiProc"]) def test_callback_gantt(tmpdir, plugin): import logging - import logging.handlers from os import path From d76af5773e1074cd32d4d7fa6d46e91523fbd81a Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Thu, 1 Apr 2021 12:17:20 -0400 Subject: [PATCH 09/22] =?UTF-8?q?FIX:=20next=20=E2=89=A0=20continue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ref https://github.com/nipy/nipype/pull/3290#discussion_r605706537, https://github.com/nipy/nipype/pull/3290#discussion_r605711954 Co-authored-by: Mathias Goncalves --- nipype/utils/draw_gantt_chart.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index c7a1a5153f..fe6cc7626d 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -159,14 +159,14 @@ def calculate_resource_timeseries(events, resource): try: all_res += float(event[resource]) except ValueError: - next + continue current_time = event["start"] elif event["event"] == "finish": if resource in event: try: all_res -= float(event[resource]) except ValueError: - next + continue current_time = event["finish"] res[current_time] = all_res From a80923f6e1a6e7a0fd09ae6c410b583900d2093f Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Mon, 5 Apr 2021 09:37:05 -0400 Subject: [PATCH 10/22] TST: Skip test that requires pandas if pandas not installed Co-authored-by: Chris Markiewicz --- nipype/pipeline/plugins/tests/test_callback.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nipype/pipeline/plugins/tests/test_callback.py b/nipype/pipeline/plugins/tests/test_callback.py index 02234522fa..5c82f11343 100644 --- a/nipype/pipeline/plugins/tests/test_callback.py +++ b/nipype/pipeline/plugins/tests/test_callback.py @@ -63,6 +63,7 @@ def test_callback_exception(tmpdir, plugin, stop_on_first_crash): @pytest.mark.parametrize("plugin", ["Linear", "MultiProc", "LegacyMultiProc"]) +@pytest.mark.skipif(not has_pandas, "Test requires pandas") def test_callback_gantt(tmpdir, plugin): import logging From 9096a5be85909d4570491c18c42629e17645efd3 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 13 Apr 2021 10:02:19 -0400 Subject: [PATCH 11/22] TEST: Add pandas import check --- nipype/pipeline/plugins/tests/test_callback.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nipype/pipeline/plugins/tests/test_callback.py b/nipype/pipeline/plugins/tests/test_callback.py index 5c82f11343..d2e0d26be8 100644 --- a/nipype/pipeline/plugins/tests/test_callback.py +++ b/nipype/pipeline/plugins/tests/test_callback.py @@ -7,6 +7,11 @@ import nipype.interfaces.utility as niu import nipype.pipeline.engine as pe +try: + import pandas + has_pandas = True +except ImportError: + has_pandas = False def func(): return From b1690d5beb391e08c1e5463f1e3c641cf1e9f58e Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 13 Apr 2021 10:16:17 -0400 Subject: [PATCH 12/22] STY: black --- nipype/pipeline/plugins/tests/test_callback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/plugins/tests/test_callback.py b/nipype/pipeline/plugins/tests/test_callback.py index d2e0d26be8..66526c76c4 100644 --- a/nipype/pipeline/plugins/tests/test_callback.py +++ b/nipype/pipeline/plugins/tests/test_callback.py @@ -11,7 +11,7 @@ import pandas has_pandas = True except ImportError: - has_pandas = False + has_pandas = False def func(): return From de6657e1ebdde1c6ed8c2c2914dfca70f5de7358 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 13 Apr 2021 10:34:55 -0400 Subject: [PATCH 13/22] STY/TEST: black and skipif syntax --- nipype/pipeline/plugins/tests/test_callback.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nipype/pipeline/plugins/tests/test_callback.py b/nipype/pipeline/plugins/tests/test_callback.py index 66526c76c4..af6cbc76a1 100644 --- a/nipype/pipeline/plugins/tests/test_callback.py +++ b/nipype/pipeline/plugins/tests/test_callback.py @@ -9,10 +9,12 @@ try: import pandas + has_pandas = True except ImportError: has_pandas = False + def func(): return @@ -68,7 +70,7 @@ def test_callback_exception(tmpdir, plugin, stop_on_first_crash): @pytest.mark.parametrize("plugin", ["Linear", "MultiProc", "LegacyMultiProc"]) -@pytest.mark.skipif(not has_pandas, "Test requires pandas") +@pytest.mark.skipif(not has_pandas, reason="Test requires pandas") def test_callback_gantt(tmpdir, plugin): import logging From 6830e3ac4c5062b700a895e267c15f4aaf2e9ae1 Mon Sep 17 00:00:00 2001 From: Jon Cluce Date: Mon, 18 Nov 2024 09:55:18 -0500 Subject: [PATCH 14/22] STY: Fix typo (co{^n}vert) --- nipype/utils/draw_gantt_chart.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index fe6cc7626d..78dc589859 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -534,13 +534,15 @@ def generate_gantt_chart( # Read in json-log to get list of node dicts nodes_list = log_to_dict(logfile) - # Only include nodes with timing information, and covert timestamps + # Only include nodes with timing information, and convert timestamps # from strings to datetimes nodes_list = [ { - k: datetime.datetime.strptime(i[k], "%Y-%m-%dT%H:%M:%S.%f") - if k in {"start", "finish"} - else i[k] + k: ( + datetime.datetime.strptime(i[k], "%Y-%m-%dT%H:%M:%S.%f") + if k in {"start", "finish"} + else i[k] + ) for k in i } for i in nodes_list From 376d6e22fcc776a1c3bba297fbdf1e0dff4d7a57 Mon Sep 17 00:00:00 2001 From: Jon Cluce Date: Mon, 18 Nov 2024 11:02:12 -0500 Subject: [PATCH 15/22] FIX: Don't try to `strptime` something that's already a `datetime` --- nipype/utils/draw_gantt_chart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index 78dc589859..21e449d333 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -540,7 +540,7 @@ def generate_gantt_chart( { k: ( datetime.datetime.strptime(i[k], "%Y-%m-%dT%H:%M:%S.%f") - if k in {"start", "finish"} + if k in {"start", "finish"} and isinstance(i[k], str) else i[k] ) for k in i From 19a03554486385334babf85511deccb04665c289 Mon Sep 17 00:00:00 2001 From: Jon Cluce Date: Mon, 18 Nov 2024 11:27:58 -0500 Subject: [PATCH 16/22] TEST: Update Gantt chart tests for coverage --- .../pipeline/plugins/tests/test_callback.py | 33 ++++++++++++++----- nipype/utils/draw_gantt_chart.py | 24 ++++++++++---- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/nipype/pipeline/plugins/tests/test_callback.py b/nipype/pipeline/plugins/tests/test_callback.py index af6cbc76a1..246f2b8ecf 100644 --- a/nipype/pipeline/plugins/tests/test_callback.py +++ b/nipype/pipeline/plugins/tests/test_callback.py @@ -1,8 +1,9 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for workflow callbacks -""" +"""Tests for workflow callbacks.""" +from pathlib import Path from time import sleep +import json import pytest import nipype.interfaces.utility as niu import nipype.pipeline.engine as pe @@ -71,7 +72,7 @@ def test_callback_exception(tmpdir, plugin, stop_on_first_crash): @pytest.mark.parametrize("plugin", ["Linear", "MultiProc", "LegacyMultiProc"]) @pytest.mark.skipif(not has_pandas, reason="Test requires pandas") -def test_callback_gantt(tmpdir, plugin): +def test_callback_gantt(tmp_path: Path, plugin: str) -> None: import logging from os import path @@ -79,14 +80,14 @@ def test_callback_gantt(tmpdir, plugin): from nipype.utils.profiler import log_nodes_cb from nipype.utils.draw_gantt_chart import generate_gantt_chart - log_filename = path.join(tmpdir, "callback.log") + log_filename = tmp_path / "callback.log" logger = logging.getLogger("callback") logger.setLevel(logging.DEBUG) handler = logging.FileHandler(log_filename) logger.addHandler(handler) # create workflow - wf = pe.Workflow(name="test", base_dir=tmpdir.strpath) + wf = pe.Workflow(name="test", base_dir=str(tmp_path)) f_node = pe.Node( niu.Function(function=func, input_names=[], output_names=[]), name="f_node" ) @@ -98,7 +99,21 @@ def test_callback_gantt(tmpdir, plugin): plugin_args["n_procs"] = 8 wf.run(plugin=plugin, plugin_args=plugin_args) - generate_gantt_chart( - path.join(tmpdir, "callback.log"), 1 if plugin == "Linear" else 8 - ) - assert path.exists(path.join(tmpdir, "callback.log.html")) + with open(log_filename, "r") as _f: + loglines = _f.readlines() + + # test missing duration + first_line = json.loads(loglines[0]) + if "duration" in first_line: + del first_line["duration"] + loglines[0] = f"{json.dumps(first_line)}\n" + + # test duplicate timestamp warning + loglines.append(loglines[-1]) + + with open(log_filename, "w") as _f: + _f.write("".join(loglines)) + + with pytest.warns(Warning): + generate_gantt_chart(str(log_filename), 1 if plugin == "Linear" else 8) + assert (tmp_path / "callback.log.html").exists() diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index 21e449d333..92d9bc363c 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -102,15 +102,25 @@ def log_to_dict(logfile): nodes_list = [json.loads(l) for l in lines] - def _convert_string_to_datetime(datestring): - try: + def _convert_string_to_datetime( + datestring: str | datetime.datetime, + ) -> datetime.datetime: + """Convert a date string to a datetime object.""" + if isinstance(datestring, datetime.datetime): + datetime_object = datestring + elif isinstance(datestring, str): + date_format = ( + "%Y-%m-%dT%H:%M:%S.%f%z" + if "+" in datestring + else "%Y-%m-%dT%H:%M:%S.%f" + ) datetime_object: datetime.datetime = datetime.datetime.strptime( - datestring, "%Y-%m-%dT%H:%M:%S.%f" + datestring, date_format ) - return datetime_object - except Exception as _: - pass - return datestring + else: + msg = f"{datestring} is not a string or datetime object." + raise TypeError(msg) + return datetime_object date_object_node_list: list = list() for n in nodes_list: From 73f657f09fa7bda5cb4030f2bbe07eed38cfba2e Mon Sep 17 00:00:00 2001 From: Jon Cluce Date: Mon, 18 Nov 2024 13:17:00 -0500 Subject: [PATCH 17/22] REF: Require Pandas for tests Ref https://github.com/nipy/nipype/pull/3290#discussion_r1846980527 --- nipype/info.py | 12 +++++++----- nipype/pipeline/plugins/tests/test_callback.py | 8 -------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/nipype/info.py b/nipype/info.py index bce47c3e3a..84b84d34ad 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -99,11 +99,12 @@ def get_nipype_gitversion(): """ # versions -NIBABEL_MIN_VERSION = "3.0" -NETWORKX_MIN_VERSION = "2.5" -NUMPY_MIN_VERSION = "1.21" -SCIPY_MIN_VERSION = "1.8" -TRAITS_MIN_VERSION = "6.2" +NIBABEL_MIN_VERSION = "2.1.0" +NETWORKX_MIN_VERSION = "2.0" +NUMPY_MIN_VERSION = "1.17" +NUMPY_MAX_VERSION = "2.0" +SCIPY_MIN_VERSION = "0.14" +TRAITS_MIN_VERSION = "4.6" DATEUTIL_MIN_VERSION = "2.2" SIMPLEJSON_MIN_VERSION = "3.8.0" PROV_MIN_VERSION = "1.5.2" @@ -153,6 +154,7 @@ def get_nipype_gitversion(): TESTS_REQUIRES = [ "coverage >= 5.2.1", + "pandas > 1.5.0, <= 2.0", "pytest >= 6", "pytest-cov >=2.11", "pytest-env", diff --git a/nipype/pipeline/plugins/tests/test_callback.py b/nipype/pipeline/plugins/tests/test_callback.py index 246f2b8ecf..f5240043a2 100644 --- a/nipype/pipeline/plugins/tests/test_callback.py +++ b/nipype/pipeline/plugins/tests/test_callback.py @@ -8,13 +8,6 @@ import nipype.interfaces.utility as niu import nipype.pipeline.engine as pe -try: - import pandas - - has_pandas = True -except ImportError: - has_pandas = False - def func(): return @@ -71,7 +64,6 @@ def test_callback_exception(tmpdir, plugin, stop_on_first_crash): @pytest.mark.parametrize("plugin", ["Linear", "MultiProc", "LegacyMultiProc"]) -@pytest.mark.skipif(not has_pandas, reason="Test requires pandas") def test_callback_gantt(tmp_path: Path, plugin: str) -> None: import logging From 8329d088c97c24bf904eb662736ec2f4f3d7bd8b Mon Sep 17 00:00:00 2001 From: Jon Cluce Date: Mon, 18 Nov 2024 13:18:15 -0500 Subject: [PATCH 18/22] REF: 3.9-friendly typing.Union --- nipype/utils/draw_gantt_chart.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index 92d9bc363c..393d7f7308 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -8,6 +8,7 @@ import random import datetime import simplejson as json +from typing import Union from collections import OrderedDict from warnings import warn @@ -103,7 +104,7 @@ def log_to_dict(logfile): nodes_list = [json.loads(l) for l in lines] def _convert_string_to_datetime( - datestring: str | datetime.datetime, + datestring: Union[str, datetime.datetime], ) -> datetime.datetime: """Convert a date string to a datetime object.""" if isinstance(datestring, datetime.datetime): From 4c0835f608e9e2fd5861aab947fde6577a58d3e8 Mon Sep 17 00:00:00 2001 From: Jon Cluce Date: Mon, 18 Nov 2024 13:51:04 -0500 Subject: [PATCH 19/22] REF: Handle absence/presence of tzinfo --- nipype/info.py | 11 +++++------ nipype/utils/draw_gantt_chart.py | 9 ++++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/nipype/info.py b/nipype/info.py index 84b84d34ad..f7ca6e66a9 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -99,12 +99,11 @@ def get_nipype_gitversion(): """ # versions -NIBABEL_MIN_VERSION = "2.1.0" -NETWORKX_MIN_VERSION = "2.0" -NUMPY_MIN_VERSION = "1.17" -NUMPY_MAX_VERSION = "2.0" -SCIPY_MIN_VERSION = "0.14" -TRAITS_MIN_VERSION = "4.6" +NIBABEL_MIN_VERSION = "3.0" +NETWORKX_MIN_VERSION = "2.5" +NUMPY_MIN_VERSION = "1.21" +SCIPY_MIN_VERSION = "1.8" +TRAITS_MIN_VERSION = "6.2" DATEUTIL_MIN_VERSION = "2.2" SIMPLEJSON_MIN_VERSION = "3.8.0" PROV_MIN_VERSION = "1.5.2" diff --git a/nipype/utils/draw_gantt_chart.py b/nipype/utils/draw_gantt_chart.py index 393d7f7308..64a0d793db 100644 --- a/nipype/utils/draw_gantt_chart.py +++ b/nipype/utils/draw_gantt_chart.py @@ -302,7 +302,14 @@ def draw_nodes(start, nodes_list, cores, minute_scale, space_between_minutes, co # Left left = 60 for core in range(len(end_times)): - if end_times[core] < node_start: + try: + end_time_condition = end_times[core] < node_start + except TypeError: + # if one has a timezone and one does not + end_time_condition = end_times[core].replace( + tzinfo=None + ) < node_start.replace(tzinfo=None) + if end_time_condition: left += core * 30 end_times[core] = datetime.datetime( node_finish.year, From 12b6e3732ae4a9a97cb65db90a64ed6c6c31bbfd Mon Sep 17 00:00:00 2001 From: Jon Cluce Date: Mon, 18 Nov 2024 14:13:38 -0500 Subject: [PATCH 20/22] FIX: Drop pandas ceiling --- nipype/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/info.py b/nipype/info.py index f7ca6e66a9..c8a8b9686d 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -153,7 +153,7 @@ def get_nipype_gitversion(): TESTS_REQUIRES = [ "coverage >= 5.2.1", - "pandas > 1.5.0, <= 2.0", + "pandas > 1.5.0", "pytest >= 6", "pytest-cov >=2.11", "pytest-env", From a693e129dc1fe40297d3fc4e0c228dcb41e184d4 Mon Sep 17 00:00:00 2001 From: Jon Cluce Date: Mon, 18 Nov 2024 14:27:34 -0500 Subject: [PATCH 21/22] =?UTF-8?q?REF:=20=E2=89=A5=201.5.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Chris Markiewicz --- nipype/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/info.py b/nipype/info.py index c8a8b9686d..47d765b34e 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -153,7 +153,7 @@ def get_nipype_gitversion(): TESTS_REQUIRES = [ "coverage >= 5.2.1", - "pandas > 1.5.0", + "pandas >= 1.5.0", "pytest >= 6", "pytest-cov >=2.11", "pytest-env", From 72239141a9d6b2d684d44447aec188372091ccc6 Mon Sep 17 00:00:00 2001 From: Jon Cluce Date: Mon, 18 Nov 2024 14:27:56 -0500 Subject: [PATCH 22/22] FIX: Too much indentation Co-authored-by: Chris Markiewicz --- nipype/pipeline/plugins/tests/test_callback.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nipype/pipeline/plugins/tests/test_callback.py b/nipype/pipeline/plugins/tests/test_callback.py index f5240043a2..b10238ec4a 100644 --- a/nipype/pipeline/plugins/tests/test_callback.py +++ b/nipype/pipeline/plugins/tests/test_callback.py @@ -94,14 +94,14 @@ def test_callback_gantt(tmp_path: Path, plugin: str) -> None: with open(log_filename, "r") as _f: loglines = _f.readlines() - # test missing duration - first_line = json.loads(loglines[0]) - if "duration" in first_line: - del first_line["duration"] - loglines[0] = f"{json.dumps(first_line)}\n" - - # test duplicate timestamp warning - loglines.append(loglines[-1]) + # test missing duration + first_line = json.loads(loglines[0]) + if "duration" in first_line: + del first_line["duration"] + loglines[0] = f"{json.dumps(first_line)}\n" + + # test duplicate timestamp warning + loglines.append(loglines[-1]) with open(log_filename, "w") as _f: _f.write("".join(loglines))