diff --git a/src/pint/toa.py b/src/pint/toa.py index a2e523c27..b9c5a12c6 100644 --- a/src/pint/toa.py +++ b/src/pint/toa.py @@ -3,12 +3,12 @@ In particular, single TOAs are represented by :class:`pint.toa.TOA` objects, and if you want to manage a collection of these we recommend you use a :class:`pint.toa.TOAs` object as this makes certain operations much more convenient. - """ from __future__ import absolute_import, division, print_function, unicode_literals import copy import gzip +import hashlib import os import re import warnings @@ -78,6 +78,12 @@ JD_MJD = 2400000.5 +def _compute_hash(filename): + h = hashlib.sha256() + h.update(open(filename, "rb").read()) + return h.digest() + + def get_TOAs( timfile, ephem=None, @@ -149,7 +155,6 @@ def get_TOAs( ------- TOAs Completed TOAs object representing the data. - """ if model: # If the keyword args are set, override what is in the model @@ -173,11 +178,13 @@ def get_TOAs( ) else: log.warning( - f'CLOCK = {model["CLOCK"].value} is not implemented. Using TT({bipm_default}) instead.' + f'CLOCK = {model["CLOCK"].value} is not implemented. ' + f"Using TT({bipm_default}) instead." ) else: log.warning( - f'CLOCK = {model["CLOCK"].value} is not implemented. Using TT({bipm_default}) instead.' + f'CLOCK = {model["CLOCK"].value} is not implemented. ' + f"Using TT({bipm_default}) instead." ) if planets is None and model["PLANET_SHAPIRO"].value: @@ -190,6 +197,14 @@ def get_TOAs( picklefile = _check_pickle(timfile) if picklefile: t = TOAs(picklefile) + if hasattr(t, "hashes"): + for f, v in t.hashes.items(): + if _compute_hash(f) != v: + updatepickle = True + log.info("Pickle based on files that have changed") + break + else: + updatepickle = True if ( include_gps is not None and t.clock_corr_info.get("include_gps", None) != include_gps @@ -252,6 +267,11 @@ def get_TOAs( if usepickle and updatepickle: log.info("Pickling TOAs.") + if isinstance(t.filename, str): + files = [t.filename] + else: + files = t.filename + t.hashes = {f: _compute_hash(f) for f in files} t.pickle() return t @@ -278,14 +298,6 @@ def _check_pickle(toafilename, picklefilename=None): if picklefilename is None: return "" - # Check if TOA is newer than pickle - if os.path.getmtime(picklefilename) < os.path.getmtime(toafilename): - return "" - - # TODO add more tests. Some things to consider: - # 1. Check file contents via md5sum (will require storing this in pickle). - # 2. Check INCLUDEd TOA files (will require some TOA file parsing). - # All checks passed, return name of pickle. return picklefilename @@ -353,7 +365,6 @@ def _parse_TOA_line(line, fmt="Unknown"): Return an MJD tuple and a dictionary of other TOA information. The format can be one of: Comment, Command, Blank, Tempo2, Princeton, ITOA, Parkes, or Unknown. - """ MJD = None fmt = _toa_format(line, fmt) @@ -565,7 +576,7 @@ def format_toa_line( def make_fake_toas( startMJD, endMJD, ntoas, model, freq=999999, obs="GBT", error=1 * u.us ): - """Make evenly spaced toas with residuals = 0 and without errors + """Make evenly spaced toas with residuals = 0 and without errors. Might be able to do different frequencies if fed an array of frequencies, only works with one observatory at a time @@ -960,7 +971,7 @@ def __init__(self, toafile=None, toalist=None): inc_fns = [ x[0][1] for x in self.commands if x[0][0].upper() == "INCLUDE" ] - self.filename = [toafile] + inc_fns if inc_fns else toafile + self.filename = [toafile] + inc_fns if inc_fns else toafile elif toafile is not None: self.read_toa_file(toafile) self.filename = "" @@ -1268,7 +1279,7 @@ def unselect(self): ) try: self.table = self.table_selects.pop() - except (AttributeError, IndexError) as e: + except (AttributeError, IndexError): log.error("No previous TOA table found. No changes made.") def pickle(self, filename=None): @@ -1277,8 +1288,12 @@ def pickle(self, filename=None): self.pintversion = pint.__version__ if filename is not None: pickle.dump(self, open(filename, "wb")) - elif (self.filename is not None) and (type(self.filename) is not list): - pickle.dump(self, gzip.open(self.filename + ".pickle.gz", "wb")) + elif self.filename is not None: + if isinstance(self.filename, str): + filename = self.filename + else: + filename = self.filename[0] + pickle.dump(self, gzip.open(filename + ".pickle.gz", "wb")) else: raise ValueError("TOA pickle method needs a (single) filename.") @@ -1781,9 +1796,8 @@ def read_pickle_file(self, filename): tmp = pickle.load(infile) if not hasattr(tmp, "pintversion") or tmp.pintversion != pint.__version__: log.error( - "PINT version in pickle file is different than current version!\n*** Suggest deleting {}".format( - filename - ) + f"PINT version in pickle file is different than current version! " + f"*** Suggest deleting {filename}" ) self.filename = tmp.filename if hasattr(tmp, "toas"): @@ -1794,6 +1808,7 @@ def read_pickle_file(self, filename): self.clock_corr_info = tmp.clock_corr_info self.ephem = tmp.ephem self.planets = tmp.planets + self.hashes = tmp.hashes def read_toa_file(self, filename, process_includes=True, top=True): """Read TOAs from the given filename. @@ -1927,7 +1942,7 @@ def merge_TOAs(TOAs_list): """Merge a list of TOAs instances and return a new combined TOAs instance In order for a merge to work, each TOAs instance needs to have - been created using the same Solar System Ephemeris (EPHEM), + been created using the same Solar System Ephemeris (EPHEM), the same reference timescale (i.e. CLOCK), and the same value of .planets (i.e. whether planetary PosVel columns are in the tables or not). @@ -1990,4 +2005,3 @@ def merge_TOAs(TOAs_list): # Now we need to re-arrange and group the tables nt.table = nt.table.group_by("obs") return nt - diff --git a/tests/datafile/parkes.toa b/tests/datafile/parkes.toa index c36497b34..6e4c8722f 100644 --- a/tests/datafile/parkes.toa +++ b/tests/datafile/parkes.toa @@ -1,9 +1,9 @@ # These are Parkes format and have been changed to also be Parkes observatory - PUPPI_J2044+28_58852_652 432.3420 58852.7590686063892 0.00 120.75 7 - PUPPI_J2044+28_58852_679 1433.237 58852.7921241426833 0.00 248.48 7 - PUPPI_J2044+28_58882_586 432.3420 58882.6825930247209 0.00 219.67 7 - PUPPI_J2044+28_58968_366 432.3430 58968.4254960153599 0.00 167.32 7 - PUPPI_J2044+28_58970_343 431.9220 58970.4001441717446 0.00 136.16 7 - PUPPI_J2044+28_58974_334 431.9220 58974.3901323879601 0.00 132.61 7 - PUPPI_J2044+28_58981_310 431.9220 58981.3623757223578 0.00 154.07 7 - PUPPI_J2044+28_59000_285 431.9220 59000.3318112360150 0.00 180.83 7 + PUPPI_J2044+28_58852_652 432.3420 58852.7590686063892 0.00 120.75 @ + PUPPI_J2044+28_58852_679 1433.237 58852.7921241426833 0.00 248.48 @ + PUPPI_J2044+28_58882_586 432.3420 58882.6825930247209 0.00 219.67 @ + PUPPI_J2044+28_58968_366 432.3430 58968.4254960153599 0.00 167.32 @ + PUPPI_J2044+28_58970_343 431.9220 58970.4001441717446 0.00 136.16 @ + PUPPI_J2044+28_58974_334 431.9220 58974.3901323879601 0.00 132.61 @ + PUPPI_J2044+28_58981_310 431.9220 58981.3623757223578 0.00 154.07 @ + PUPPI_J2044+28_59000_285 431.9220 59000.3318112360150 0.00 180.83 @ diff --git a/tests/test_toa_reader.py b/tests/test_toa_reader.py index 574d45210..1c0fed223 100644 --- a/tests/test_toa_reader.py +++ b/tests/test_toa_reader.py @@ -45,7 +45,7 @@ def setUp(self): def test_read_parkes(self): ts = toa.get_TOAs("parkes.toa") - assert "parkes" in ts.observatories + assert "barycenter" in ts.observatories assert ts.ntoas == 8 def test_commands(self): @@ -158,4 +158,3 @@ def test_toa_merge(): toas[0].ephem = "DE436" with pytest.raises(TypeError): nt = toa.merge_TOAs(toas) -