From 17d106445b1a37c76a6f23860f21d7b52bf76676 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 6 Nov 2020 12:41:02 -0800 Subject: [PATCH 01/40] remove read() deprecation warning, add diff_freq extra keyword to indicate when diff_freq on load has been done --- SSINS/sky_subtract.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index 5ced6f51..d099d839 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -42,8 +42,6 @@ def read(self, filename, diff=False, flag_choice=None, INS=None, custom=None, custom: A custom flag array for apply_flags() kwargs: Additional kwargs are passed to UVData.read() """ - warnings.warn("SS.read will be renamed to SS.read_data soon to avoid" - " conflicts with UVData.read.", category=PendingDeprecationWarning) super().read(filename, **kwargs) @@ -51,6 +49,7 @@ def read(self, filename, diff=False, flag_choice=None, INS=None, custom=None, if diff: self.diff() self.apply_flags(flag_choice=flag_choice, INS=INS, custom=custom) + self.extra_keywords.update(diff_freq=True) else: # This warning will be issued when diff is False and there is some data read in # If filename is a list of files, then this warning will get issued in the recursive call in UVData.read From 305f31c12b4695d35e88b706aee0ddf4fc15b13b Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 6 Nov 2020 13:17:34 -0800 Subject: [PATCH 02/40] shorten keyword to 8 chars per standard --- SSINS/sky_subtract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index d099d839..91dec4f7 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -49,7 +49,7 @@ def read(self, filename, diff=False, flag_choice=None, INS=None, custom=None, if diff: self.diff() self.apply_flags(flag_choice=flag_choice, INS=INS, custom=custom) - self.extra_keywords.update(diff_freq=True) + self.extra_keywords.update(dif_freq=True) else: # This warning will be issued when diff is False and there is some data read in # If filename is a list of files, then this warning will get issued in the recursive call in UVData.read From 558f8c6145808fe1c2a69f1ce96670755e042bcc Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 11 Dec 2020 12:36:29 -0800 Subject: [PATCH 03/40] trigger dif_freq write in all cases && add test cases --- SSINS/sky_subtract.py | 3 +++ SSINS/tests/test_SS.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index 91dec4f7..7e970495 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -56,9 +56,12 @@ def read(self, filename, diff=False, flag_choice=None, INS=None, custom=None, warnings.warn("diff on read defaults to False now. Please double" " check SS.read call and ensure the appropriate" " keyword arguments for your intended use case.") + self.extra_keywords.update(dif_freq=False) if flag_choice is not None: warnings.warn("flag_choice will be ignored on read since" " diff is being skipped.") + else: + self.extra_keywords.update(dif_freq=False) def apply_flags(self, flag_choice=None, INS=None, custom=None): """ diff --git a/SSINS/tests/test_SS.py b/SSINS/tests/test_SS.py index b73553d3..67db72a4 100644 --- a/SSINS/tests/test_SS.py +++ b/SSINS/tests/test_SS.py @@ -22,6 +22,9 @@ def test_SS_read(): ss.read(testfile, read_data=False) assert ss.data_array is None, "Data array is not None" + # See that it is not yet flagged as diffed + assert ss.extra_keywords["dif_freq"] != True + # Test select on read and diff ss.read(testfile, times=np.unique(ss.time_array)[1:10], diff=True) assert ss.Ntimes == 8, "Number of times after diff disagrees!" @@ -64,6 +67,8 @@ def test_diff(): assert np.all(ss.uvw_array == diff_uvw), "uvw_arrays disagree!" assert np.all(ss.ant_1_array == np.array([0, 0])), "ant_1_array disagrees!" assert np.all(ss.ant_2_array == np.array([1, 2])), "ant_2_array disagrees!" + assert ss.extra_keywords["dif_freq"] is not None + assert ss.extra_keywords["dif_freq"] == True @pytest.mark.filterwarnings("ignore:SS.read", "ignore:Reordering") From b673059a82a98d0b833f494bc8a145fa6d8355d8 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 11 Dec 2020 12:47:23 -0800 Subject: [PATCH 04/40] properly use 'is' in dif_freq test singletons should be compared with is, not == --- SSINS/tests/test_SS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SSINS/tests/test_SS.py b/SSINS/tests/test_SS.py index 67db72a4..fe007f99 100644 --- a/SSINS/tests/test_SS.py +++ b/SSINS/tests/test_SS.py @@ -23,7 +23,7 @@ def test_SS_read(): assert ss.data_array is None, "Data array is not None" # See that it is not yet flagged as diffed - assert ss.extra_keywords["dif_freq"] != True + assert ss.extra_keywords["dif_freq"] is False # Test select on read and diff ss.read(testfile, times=np.unique(ss.time_array)[1:10], diff=True) @@ -68,7 +68,7 @@ def test_diff(): assert np.all(ss.ant_1_array == np.array([0, 0])), "ant_1_array disagrees!" assert np.all(ss.ant_2_array == np.array([1, 2])), "ant_2_array disagrees!" assert ss.extra_keywords["dif_freq"] is not None - assert ss.extra_keywords["dif_freq"] == True + assert ss.extra_keywords["dif_freq"] is True @pytest.mark.filterwarnings("ignore:SS.read", "ignore:Reordering") From 94193d0d88a4832542e000a243e24a8bb56ba731 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 11 Dec 2020 12:52:36 -0800 Subject: [PATCH 05/40] add INS extra keywords test no implementation yet--test driven development --- SSINS/tests/test_INS.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/SSINS/tests/test_INS.py b/SSINS/tests/test_INS.py index 6845a675..1c1eed0d 100644 --- a/SSINS/tests/test_INS.py +++ b/SSINS/tests/test_INS.py @@ -35,6 +35,23 @@ def test_init(): # Check that the weights summed correctly assert np.all(test_weights == ins.weights_array), "Weights did not sum properly" +def test_extra_keywords(): + obs = '1061313128_99bl_1pol_half_time' + testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) + file_type = 'uvfits' + + ss = SS() + ss.read(testfile, flag_choice='original', diff=True) + ins = INS(ss) + + assert ins.extra_keywords["dif_freq"] is True + + ss.read(testfile, flag_choice='original', diff=False) + ins = INS(ss) + + assert ins.extra_keywords["dif_freq"] is False + + def test_no_diff_start(): obs = '1061313128_99bl_1pol_half_time' From 3b31495249d23acf2eade3d43c30c956ec3091de Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Mon, 8 Feb 2021 14:05:41 -0800 Subject: [PATCH 06/40] add diff_freq test --- SSINS/tests/test_SS.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/SSINS/tests/test_SS.py b/SSINS/tests/test_SS.py index fe007f99..2e887643 100644 --- a/SSINS/tests/test_SS.py +++ b/SSINS/tests/test_SS.py @@ -9,8 +9,11 @@ Tests the various capabilities of the sky_subtract class """ +<<<<<<< HEAD @pytest.mark.filterwarnings("ignore:Reordering", "ignore:SS.read") +======= +>>>>>>> add diff_freq test def test_SS_read(): obs = '1061313128_99bl_1pol_half_time' testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) @@ -67,9 +70,28 @@ def test_diff(): assert np.all(ss.uvw_array == diff_uvw), "uvw_arrays disagree!" assert np.all(ss.ant_1_array == np.array([0, 0])), "ant_1_array disagrees!" assert np.all(ss.ant_2_array == np.array([1, 2])), "ant_2_array disagrees!" - assert ss.extra_keywords["dif_freq"] is not None - assert ss.extra_keywords["dif_freq"] is True +def test_diff_freq(): + obs = '1061313128_99bl_1pol_half_time' + testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) + + ss = SS() + uv = UVData() + + # Read in two times and two baselines of data, so that the diff is obvious. + uv.read(testfile, read_data=False) + times = np.unique(uv.time_array)[:2] + bls = [(0, 1), (0, 2)] + uv.read(testfile, times=times, bls=bls) + uv.reorder_blts(order='baseline') + + diff_dat = np.diff(uv.data_array, axis=2) + + with pytest.warns(UserWarning, match="Reordering data array to baseline order to perform differencing."): + ss.read(testfile, diff=False, diff_freq=True, times=times, bls=bls) + print(ss._data_array.form) + #ss.reorder_blts(order='baseline') + assert np.all(ss.data_array == diff_dat), "Data values are different!" @pytest.mark.filterwarnings("ignore:SS.read", "ignore:Reordering") def test_apply_flags(): From b10a0f32b149ee4989502cd45c2c54e60f0f00bc Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Mon, 8 Feb 2021 14:09:28 -0800 Subject: [PATCH 07/40] add crude implementation of diff_freq --- SSINS/sky_subtract.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index 7e970495..2b56ab9a 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -26,7 +26,7 @@ def __init__(self): """Array of length Nfreqs that stores maximum likelihood estimators for each frequency, calculated using the MLE_calc method""" - def read(self, filename, diff=False, flag_choice=None, INS=None, custom=None, + def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None, custom=None, **kwargs): """ @@ -37,6 +37,7 @@ def read(self, filename, diff=False, flag_choice=None, INS=None, custom=None, Args: filename (str or list of str): The filepath(s) to read in. diff (bool): If True, and data was read in, then difference the visibilities in time + diff_freq (bool): If True, and data was read in, then difference the visibilities in freq flag_choice: Sets flags for the data array on read using apply_flags method. INS: An INS object for apply_flags() custom: A custom flag array for apply_flags() @@ -49,6 +50,9 @@ def read(self, filename, diff=False, flag_choice=None, INS=None, custom=None, if diff: self.diff() self.apply_flags(flag_choice=flag_choice, INS=INS, custom=custom) + self.extra_keywords.update(dif_time=True) + if diff_freq: + self.diff_freq() self.extra_keywords.update(dif_freq=True) else: # This warning will be issued when diff is False and there is some data read in @@ -227,6 +231,33 @@ def diff(self): super().set_lsts_from_time_array() self.data_array = np.ma.masked_array(self.data_array) + def diff_freq(self): + + """ + Differences the visibilities in freq. Does so independently for each baseline, + so different integration times or sets of time centers are supported. + The flags are propagated by taking the boolean OR of the entries that correspond + to the visibilities that are differenced from one another. Other metadata + attributes are also adjusted so that the resulting SS object passes + UVData.check() + """ + + if self.blt_order != 'baseline': + warnings.warn("Reordering data array to baseline order to perform differencing.") + self.reorder_blts(order='baseline') + + diff_dat = np.diff(self.data_array, axis=2) #axis 2: nfreqs (see uvdata object) + self.data_array = diff_dat + + print(self.data_array.shape) + """The frequency-differenced visibilities. Complex array of shape (Nblts, Nspws, Nfreqs, Npols).""" + + self.Nfreqs -= 1 + self.data_array = np.ma.masked_array(self.data_array) + self.freq_array = self.freq_array[:,1:] + self.nsample_array = self.nsample_array[:,:,1:,:] + self.flag_array = self.flag_array[:,:,1:,:] + def MLE_calc(self): """ From ad33658d3a442fe75651831a8357358951a8b666 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Tue, 16 Feb 2021 15:14:42 -0800 Subject: [PATCH 08/40] fix conditional statements to have SS pass keywords test --- SSINS/sky_subtract.py | 12 ++++++------ SSINS/tests/test_INS.py | 6 ++++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index 2b56ab9a..76481761 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -50,22 +50,22 @@ def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None if diff: self.diff() self.apply_flags(flag_choice=flag_choice, INS=INS, custom=custom) - self.extra_keywords.update(dif_time=True) + self.extra_keywords['dif_time'] = True if diff_freq: self.diff_freq() - self.extra_keywords.update(dif_freq=True) - else: + self.extra_keywords['dif_freq'] = True + if not diff_freq and not diff: # This warning will be issued when diff is False and there is some data read in # If filename is a list of files, then this warning will get issued in the recursive call in UVData.read warnings.warn("diff on read defaults to False now. Please double" " check SS.read call and ensure the appropriate" " keyword arguments for your intended use case.") - self.extra_keywords.update(dif_freq=False) + self.extra_keywords['dif_time'] = False + self.extra_keywords['dif_freq'] = False if flag_choice is not None: warnings.warn("flag_choice will be ignored on read since" " diff is being skipped.") - else: - self.extra_keywords.update(dif_freq=False) + def apply_flags(self, flag_choice=None, INS=None, custom=None): """ diff --git a/SSINS/tests/test_INS.py b/SSINS/tests/test_INS.py index 1c1eed0d..a9aadf2d 100644 --- a/SSINS/tests/test_INS.py +++ b/SSINS/tests/test_INS.py @@ -44,12 +44,14 @@ def test_extra_keywords(): ss.read(testfile, flag_choice='original', diff=True) ins = INS(ss) - assert ins.extra_keywords["dif_freq"] is True + assert ss.extra_keywords['dif_time'] is True + assert ins.extra_keywords['dif_time'] is True ss.read(testfile, flag_choice='original', diff=False) ins = INS(ss) - assert ins.extra_keywords["dif_freq"] is False + assert ss.extra_keywords['dif_time'] is False + assert ins.extra_keywords['dif_time'] is False From 02e7871b784348a51e9bbc3ccb1b10dfa6780746 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Tue, 16 Feb 2021 16:28:04 -0800 Subject: [PATCH 09/40] fix test_SS_read test by always setting extra_keywords --- SSINS/sky_subtract.py | 5 ++++- SSINS/tests/test_SS.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index 76481761..18af0ef9 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -46,6 +46,7 @@ def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None super().read(filename, **kwargs) + # always set extra keywords, else key errors when trying to check if (self.data_array is not None): if diff: self.diff() @@ -65,7 +66,9 @@ def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None if flag_choice is not None: warnings.warn("flag_choice will be ignored on read since" " diff is being skipped.") - + else: + self.extra_keywords['dif_time'] = False + self.extra_keywords['dif_freq'] = False def apply_flags(self, flag_choice=None, INS=None, custom=None): """ diff --git a/SSINS/tests/test_SS.py b/SSINS/tests/test_SS.py index 2e887643..d13d4df8 100644 --- a/SSINS/tests/test_SS.py +++ b/SSINS/tests/test_SS.py @@ -26,7 +26,7 @@ def test_SS_read(): assert ss.data_array is None, "Data array is not None" # See that it is not yet flagged as diffed - assert ss.extra_keywords["dif_freq"] is False + assert ss.extra_keywords['dif_freq'] is False # Test select on read and diff ss.read(testfile, times=np.unique(ss.time_array)[1:10], diff=True) From 8932d26659aed5b26c1ece2554813c4f7405a974 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Mon, 22 Feb 2021 12:09:58 -0800 Subject: [PATCH 10/40] add test_diff_freq_mask test to test_ss --- SSINS/tests/test_SS.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/SSINS/tests/test_SS.py b/SSINS/tests/test_SS.py index d13d4df8..d3afecd2 100644 --- a/SSINS/tests/test_SS.py +++ b/SSINS/tests/test_SS.py @@ -9,11 +9,8 @@ Tests the various capabilities of the sky_subtract class """ -<<<<<<< HEAD - @pytest.mark.filterwarnings("ignore:Reordering", "ignore:SS.read") -======= ->>>>>>> add diff_freq test + def test_SS_read(): obs = '1061313128_99bl_1pol_half_time' testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) @@ -71,6 +68,7 @@ def test_diff(): assert np.all(ss.ant_1_array == np.array([0, 0])), "ant_1_array disagrees!" assert np.all(ss.ant_2_array == np.array([1, 2])), "ant_2_array disagrees!" +#checks whether diff_freq reads in and out, and the diff values are sane def test_diff_freq(): obs = '1061313128_99bl_1pol_half_time' testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) @@ -93,7 +91,26 @@ def test_diff_freq(): #ss.reorder_blts(order='baseline') assert np.all(ss.data_array == diff_dat), "Data values are different!" +<<<<<<< HEAD @pytest.mark.filterwarnings("ignore:SS.read", "ignore:Reordering") +======= +#checks whether diff_freq masks properly +def test_diff_freq_mask(): + obs = '1061313128_99bl_1pol_half_time' + testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) + ss = SS() + + #read in test file + ss.read(testfile, read_data=False, diff=False, diff_freq=False) + ss.apply_flags(flag_choice='original') + assert ss.flag_array is not None + temp_array = np.logical_or(ss.flag_array, ss.flag_array) + nonzero_or = np.count_nonzero(temp_array) + nonzero_flags = np.count_nonzero(ss.flag_array[::2]) + assert (nonzero_or > nonzero_flags) + + +>>>>>>> add test_diff_freq_mask test to test_ss def test_apply_flags(): obs = '1061313128_99bl_1pol_half_time' From 82963542088885646d93e4c63ded65e9c74b2bee Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Mon, 22 Feb 2021 13:09:38 -0800 Subject: [PATCH 11/40] average freq array entries && or the flag array sky_subtract.py in diff_freq --- SSINS/sky_subtract.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index 18af0ef9..dff3d941 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -237,8 +237,7 @@ def diff(self): def diff_freq(self): """ - Differences the visibilities in freq. Does so independently for each baseline, - so different integration times or sets of time centers are supported. + Differences the visibilities in freq. Does so independently for each time. The flags are propagated by taking the boolean OR of the entries that correspond to the visibilities that are differenced from one another. Other metadata attributes are also adjusted so that the resulting SS object passes @@ -257,9 +256,9 @@ def diff_freq(self): self.Nfreqs -= 1 self.data_array = np.ma.masked_array(self.data_array) - self.freq_array = self.freq_array[:,1:] + self.freq_array = (self.freq_array[:,1:] + self.freq_array[:,:-1]) / 2.0 self.nsample_array = self.nsample_array[:,:,1:,:] - self.flag_array = self.flag_array[:,:,1:,:] + self.flag_array = np.logical_or(self.flag_array[:,:,1:,:], self.flag_array[:,:,:-1,:]) def MLE_calc(self): From e4872427516d892741ec03a04c58e36700212062 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Mon, 22 Feb 2021 14:07:19 -0800 Subject: [PATCH 12/40] scrub freq_mask test for ss --- SSINS/tests/test_SS.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/SSINS/tests/test_SS.py b/SSINS/tests/test_SS.py index d3afecd2..82936b0a 100644 --- a/SSINS/tests/test_SS.py +++ b/SSINS/tests/test_SS.py @@ -91,9 +91,6 @@ def test_diff_freq(): #ss.reorder_blts(order='baseline') assert np.all(ss.data_array == diff_dat), "Data values are different!" -<<<<<<< HEAD -@pytest.mark.filterwarnings("ignore:SS.read", "ignore:Reordering") -======= #checks whether diff_freq masks properly def test_diff_freq_mask(): obs = '1061313128_99bl_1pol_half_time' @@ -109,10 +106,7 @@ def test_diff_freq_mask(): nonzero_flags = np.count_nonzero(ss.flag_array[::2]) assert (nonzero_or > nonzero_flags) - ->>>>>>> add test_diff_freq_mask test to test_ss def test_apply_flags(): - obs = '1061313128_99bl_1pol_half_time' testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) file_type = 'uvfits' From 173ff52b22c580aa730104cce38c72da65f67826 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Mon, 1 Mar 2021 13:44:14 -0800 Subject: [PATCH 13/40] add ability to override extra_keywords at read() level --- SSINS/sky_subtract.py | 13 ++++++++++++- SSINS/tests/test_SS.py | 26 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index dff3d941..7c961964 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -27,7 +27,7 @@ def __init__(self): each frequency, calculated using the MLE_calc method""" def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None, custom=None, - **kwargs): + override_keyword=None, **kwargs): """ Reads in a file that is compatible with UVData object by first calling @@ -42,6 +42,7 @@ def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None INS: An INS object for apply_flags() custom: A custom flag array for apply_flags() kwargs: Additional kwargs are passed to UVData.read() + override_keyword (str): sets a keyword to override to True regardless of internal diffs. """ super().read(filename, **kwargs) @@ -69,6 +70,16 @@ def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None else: self.extra_keywords['dif_time'] = False self.extra_keywords['dif_freq'] = False + # hardcoded cases for override_keyword to set dif_freq/dif_time (or both) + # useful if data is pre-differenced rather than internally diff-ed by read() itself + if (override_keyword is not None): + if override_keyword == 'dif_time': + self.extra_keywords['dif_time'] = True + elif override_keyword == 'dif_freq': + self.extra_keywords['dif_freq'] = True + elif override_keyword == 'both': + self.extra_keywords['dif_freq'] = True + self.extra_keywords['dif_time'] = True def apply_flags(self, flag_choice=None, INS=None, custom=None): """ diff --git a/SSINS/tests/test_SS.py b/SSINS/tests/test_SS.py index 82936b0a..bd07c603 100644 --- a/SSINS/tests/test_SS.py +++ b/SSINS/tests/test_SS.py @@ -68,6 +68,32 @@ def test_diff(): assert np.all(ss.ant_1_array == np.array([0, 0])), "ant_1_array disagrees!" assert np.all(ss.ant_2_array == np.array([1, 2])), "ant_2_array disagrees!" + +def test_keyword_override_time(): + obs = '1061313128_99bl_1pol_half_time' + testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) + ss = SS() + ss.read(testfile, read_data=False, diff=False, diff_freq=False, override_keyword='dif_time') + assert ss.extra_keywords['dif_time'] is True + + +def test_keyword_override_freq(): + obs = '1061313128_99bl_1pol_half_time' + ss = SS() + testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) + ss.read(testfile, read_data=False, diff=False, diff_freq=False, override_keyword='dif_time') + assert ss.extra_keywords['dif_time'] is True + + +def test_keyword_override_both(): + obs = '1061313128_99bl_1pol_half_time' + ss = SS() + testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) + ss.read(testfile, read_data=False, diff=False, diff_freq=False, override_keyword='both') + assert ss.extra_keywords['dif_time'] is True + assert ss.extra_keywords['dif_freq'] is True + + #checks whether diff_freq reads in and out, and the diff values are sane def test_diff_freq(): obs = '1061313128_99bl_1pol_half_time' From 38199bdd80025b3994a97bd35820472540ed2b1f Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 5 Mar 2021 11:19:14 -0800 Subject: [PATCH 14/40] resolve bounced tests caused by merge --- SSINS/tests/test_SS.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/SSINS/tests/test_SS.py b/SSINS/tests/test_SS.py index bd07c603..1830e43c 100644 --- a/SSINS/tests/test_SS.py +++ b/SSINS/tests/test_SS.py @@ -17,9 +17,7 @@ def test_SS_read(): ss = SS() - # Test reading in only metadata skips if block and warning - with pytest.warns(PendingDeprecationWarning, match="SS.read will be renamed"): - ss.read(testfile, read_data=False) + ss.read(testfile, read_data=False) assert ss.data_array is None, "Data array is not None" # See that it is not yet flagged as diffed @@ -132,6 +130,7 @@ def test_diff_freq_mask(): nonzero_flags = np.count_nonzero(ss.flag_array[::2]) assert (nonzero_or > nonzero_flags) +@pytest.mark.filterwarnings("ignore:SS.read", "ignore:Reordering") def test_apply_flags(): obs = '1061313128_99bl_1pol_half_time' testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) From bf3bf8d16a3e80588d05caaa333053e7e1dac7c5 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 5 Mar 2021 11:37:28 -0800 Subject: [PATCH 15/40] fix keyword_override_freq testing time instead of freq --- SSINS/tests/test_SS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SSINS/tests/test_SS.py b/SSINS/tests/test_SS.py index 1830e43c..3396457d 100644 --- a/SSINS/tests/test_SS.py +++ b/SSINS/tests/test_SS.py @@ -79,8 +79,8 @@ def test_keyword_override_freq(): obs = '1061313128_99bl_1pol_half_time' ss = SS() testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) - ss.read(testfile, read_data=False, diff=False, diff_freq=False, override_keyword='dif_time') - assert ss.extra_keywords['dif_time'] is True + ss.read(testfile, read_data=False, diff=False, diff_freq=False, override_keyword='dif_freq') + assert ss.extra_keywords['dif_freq'] is True def test_keyword_override_both(): From ba595d938b705ec33c7538c0a41960b011c7dcd5 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Mon, 8 Mar 2021 13:56:53 -0800 Subject: [PATCH 16/40] average nsample array in diff_freq --- SSINS/sky_subtract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index 7c961964..d50b0958 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -268,7 +268,7 @@ def diff_freq(self): self.Nfreqs -= 1 self.data_array = np.ma.masked_array(self.data_array) self.freq_array = (self.freq_array[:,1:] + self.freq_array[:,:-1]) / 2.0 - self.nsample_array = self.nsample_array[:,:,1:,:] + self.nsample_array = (self.nsample_array[:,:,1:,:] + self.nsample_array[:,:,:-1,:]) / 2.0 self.flag_array = np.logical_or(self.flag_array[:,:,1:,:], self.flag_array[:,:,:-1,:]) def MLE_calc(self): From 71c664a8ee53a0312052a36b2d1b68ecabad56ee Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Tue, 9 Mar 2021 17:24:26 -0800 Subject: [PATCH 17/40] remove reorder from diff_freq --- SSINS/sky_subtract.py | 4 ---- SSINS/tests/test_SS.py | 4 +--- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index d50b0958..f9603631 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -255,10 +255,6 @@ def diff_freq(self): UVData.check() """ - if self.blt_order != 'baseline': - warnings.warn("Reordering data array to baseline order to perform differencing.") - self.reorder_blts(order='baseline') - diff_dat = np.diff(self.data_array, axis=2) #axis 2: nfreqs (see uvdata object) self.data_array = diff_dat diff --git a/SSINS/tests/test_SS.py b/SSINS/tests/test_SS.py index 3396457d..a80a1eca 100644 --- a/SSINS/tests/test_SS.py +++ b/SSINS/tests/test_SS.py @@ -105,12 +105,10 @@ def test_diff_freq(): times = np.unique(uv.time_array)[:2] bls = [(0, 1), (0, 2)] uv.read(testfile, times=times, bls=bls) - uv.reorder_blts(order='baseline') diff_dat = np.diff(uv.data_array, axis=2) - with pytest.warns(UserWarning, match="Reordering data array to baseline order to perform differencing."): - ss.read(testfile, diff=False, diff_freq=True, times=times, bls=bls) + ss.read(testfile, diff=False, diff_freq=True, times=times, bls=bls) print(ss._data_array.form) #ss.reorder_blts(order='baseline') assert np.all(ss.data_array == diff_dat), "Data values are different!" From 15088d00a081b398dbeb513705cdbccf87faab4f Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 12 Mar 2021 11:43:50 -0800 Subject: [PATCH 18/40] assert keyword values after read() with tests --- SSINS/sky_subtract.py | 2 ++ SSINS/tests/test_SS.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index f9603631..7dd6fd90 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -53,9 +53,11 @@ def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None self.diff() self.apply_flags(flag_choice=flag_choice, INS=INS, custom=custom) self.extra_keywords['dif_time'] = True + self.extra_keywords['dif_freq'] = False if diff_freq: self.diff_freq() self.extra_keywords['dif_freq'] = True + self.extra_keywords['dif_time'] = False if not diff_freq and not diff: # This warning will be issued when diff is False and there is some data read in # If filename is a list of files, then this warning will get issued in the recursive call in UVData.read diff --git a/SSINS/tests/test_SS.py b/SSINS/tests/test_SS.py index a80a1eca..eb746b0f 100644 --- a/SSINS/tests/test_SS.py +++ b/SSINS/tests/test_SS.py @@ -65,6 +65,8 @@ def test_diff(): assert np.all(ss.uvw_array == diff_uvw), "uvw_arrays disagree!" assert np.all(ss.ant_1_array == np.array([0, 0])), "ant_1_array disagrees!" assert np.all(ss.ant_2_array == np.array([1, 2])), "ant_2_array disagrees!" + assert ss.extra_keywords['dif_freq'] is False + assert ss.extra_keywords['dif_time'] is True def test_keyword_override_time(): @@ -91,7 +93,6 @@ def test_keyword_override_both(): assert ss.extra_keywords['dif_time'] is True assert ss.extra_keywords['dif_freq'] is True - #checks whether diff_freq reads in and out, and the diff values are sane def test_diff_freq(): obs = '1061313128_99bl_1pol_half_time' @@ -112,6 +113,8 @@ def test_diff_freq(): print(ss._data_array.form) #ss.reorder_blts(order='baseline') assert np.all(ss.data_array == diff_dat), "Data values are different!" + assert ss.extra_keywords['dif_freq'] is True + assert ss.extra_keywords['dif_time'] is False #checks whether diff_freq masks properly def test_diff_freq_mask(): From 3c850eeeed691429ec87489e23624de7a6164b9f Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 12 Mar 2021 12:32:49 -0800 Subject: [PATCH 19/40] add tutorial section about frequency differencing --- docs/tutorial.rst | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index b373c7f1..22924354 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -43,9 +43,20 @@ Generating the sky-subtracted visibilities >>> # The following lines make use of the time_array attribute (metadata) to >>> # read in all but the first and last integrations >>> times = np.unique(ss.time_array)[1:-1] + >>> # This read() call uses the `diff` keyword to difference the data automatically along the time axiss >>> ss.read(filepath, read_data=True, times=times, diff=True) -(c) Applying flags +(c) Differencing along the frequency axis +***************************************** +:: + >>> # We can opt to difference visibilities along the frequency axis instead of the default time axis + >>> # To do this, use the separate keyword `diff_freq` and set it to true when you call the read() function + >>> # The two modes work otherwise the same except for the axis the visibilities are differenced upon. + + >>> # Note that this will override the time-differenced data if you ran ss.read() from section (b) + >>> ss.read(filepath, read_data=True, times=times, diff_freq=True) + +(d) Applying flags ****************** :: >>> # SS.data_array is a numpy masked array. To "apply flags" is to change the mask of the data_array. @@ -72,7 +83,7 @@ Generating the sky-subtracted visibilities >>> print(np.any(ss.data_array.mask)) False -(d) Plotting using Catalog_Plot +(e) Plotting using Catalog_Plot ******************************* :: >>> from SSINS import Catalog_Plot as cp From 8fda8ef56da87262cf2ee6ddb48661d7e044dcfe Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 12 Mar 2021 12:34:40 -0800 Subject: [PATCH 20/40] add override_keyword keywords to diff() comments --- SSINS/sky_subtract.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index 7dd6fd90..a6fdc755 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -42,7 +42,8 @@ def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None INS: An INS object for apply_flags() custom: A custom flag array for apply_flags() kwargs: Additional kwargs are passed to UVData.read() - override_keyword (str): sets a keyword to override to True regardless of internal diffs. + override_keyword (str): sets a keyword to override to True regardless of internal diffs, + can be set to `dif_freq`, `dif_time`, or `both`. """ super().read(filename, **kwargs) From 2c76b054e98883d3e755168f1f96abec52f568ed Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 12 Mar 2021 13:15:39 -0800 Subject: [PATCH 21/40] remove diff_freq print statement, array comment --- SSINS/sky_subtract.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index a6fdc755..4830626b 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -261,9 +261,6 @@ def diff_freq(self): diff_dat = np.diff(self.data_array, axis=2) #axis 2: nfreqs (see uvdata object) self.data_array = diff_dat - print(self.data_array.shape) - """The frequency-differenced visibilities. Complex array of shape (Nblts, Nspws, Nfreqs, Npols).""" - self.Nfreqs -= 1 self.data_array = np.ma.masked_array(self.data_array) self.freq_array = (self.freq_array[:,1:] + self.freq_array[:,:-1]) / 2.0 From ea5060605ac634e3c6c17585462dd5f1520e5507 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 12 Mar 2021 13:22:09 -0800 Subject: [PATCH 22/40] add case for both diffs setting both extra_keywords --- SSINS/sky_subtract.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index 4830626b..9dcaaf13 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -70,6 +70,9 @@ def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None if flag_choice is not None: warnings.warn("flag_choice will be ignored on read since" " diff is being skipped.") + if diff_freq and diff: + self.extra_keywords['dif_time'] = True + self.extra_keywords['dif_freq'] = True else: self.extra_keywords['dif_time'] = False self.extra_keywords['dif_freq'] = False From b2e9d1a15e7c40984642f8dd31d756feee214cd0 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Mon, 15 Mar 2021 13:31:27 -0700 Subject: [PATCH 23/40] add freq_compattest to ins flag_uvf() --- SSINS/incoherent_noise_spectrum.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/SSINS/incoherent_noise_spectrum.py b/SSINS/incoherent_noise_spectrum.py index 4fa38a74..3da720bf 100644 --- a/SSINS/incoherent_noise_spectrum.py +++ b/SSINS/incoherent_noise_spectrum.py @@ -253,13 +253,20 @@ def flag_uvf(self, uvf, inplace=False): raise ValueError("UVFlag object must be in flag mode to write flags from INS object.") if uvf.type != 'waterfall': raise ValueError("UVFlag object must be in waterfall mode to write flags from INS object.") - try: - test_times = 0.5 * (uvf.time_array[:-1] + uvf.time_array[1:]) - time_compat = np.all(self.time_array == test_times) - assert time_compat - except Exception: - raise ValueError("UVFlag object's times do not match those of INS object.") - + if self.extra_keywords['dif_time'] = True: + try: + test_times = 0.5 * (uvf.time_array[:-1] + uvf.time_array[1:]) + time_compat = np.all(self.time_array == test_times) + assert time_compat + except Exception: + raise ValueError("UVFlag object's times do not match those of INS object.") + if self.extra_keywords['dif_freq'] = True: + try: + test_freqs = 0.5 * (uvf.freq_array[:-1] + uvf.freq_array[1:]) + freq_compat = np.all(self.freq_array == test_freqs) + assert freq_compat + except Exception: + raise ValueError("UVFlag object's times do not match those of INS object.") new_flags = self.mask_to_flags() if inplace: From e638152131c3d04cc383a48c6dd0cec6b0bebabb Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Mon, 15 Mar 2021 13:31:56 -0700 Subject: [PATCH 24/40] fix axis typo in tutorial --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 22924354..b27c7394 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -43,7 +43,7 @@ Generating the sky-subtracted visibilities >>> # The following lines make use of the time_array attribute (metadata) to >>> # read in all but the first and last integrations >>> times = np.unique(ss.time_array)[1:-1] - >>> # This read() call uses the `diff` keyword to difference the data automatically along the time axiss + >>> # This read() call uses the `diff` keyword to difference the data automatically along the time axis >>> ss.read(filepath, read_data=True, times=times, diff=True) (c) Differencing along the frequency axis From ed71833d218ab6cd3727263f5f81caf423fd6ebe Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Mon, 15 Mar 2021 14:37:35 -0700 Subject: [PATCH 25/40] use equals instead of assignment in INS flag_uvf() key checks --- SSINS/incoherent_noise_spectrum.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SSINS/incoherent_noise_spectrum.py b/SSINS/incoherent_noise_spectrum.py index 3da720bf..251ad3cb 100644 --- a/SSINS/incoherent_noise_spectrum.py +++ b/SSINS/incoherent_noise_spectrum.py @@ -253,14 +253,14 @@ def flag_uvf(self, uvf, inplace=False): raise ValueError("UVFlag object must be in flag mode to write flags from INS object.") if uvf.type != 'waterfall': raise ValueError("UVFlag object must be in waterfall mode to write flags from INS object.") - if self.extra_keywords['dif_time'] = True: + if self.extra_keywords['dif_time'] == True: try: test_times = 0.5 * (uvf.time_array[:-1] + uvf.time_array[1:]) time_compat = np.all(self.time_array == test_times) assert time_compat except Exception: raise ValueError("UVFlag object's times do not match those of INS object.") - if self.extra_keywords['dif_freq'] = True: + if self.extra_keywords['dif_freq'] == True: try: test_freqs = 0.5 * (uvf.freq_array[:-1] + uvf.freq_array[1:]) freq_compat = np.all(self.freq_array == test_freqs) From 3c5e0ab8b2222a410ce5a875ad7f1e83cc08564c Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Mon, 15 Mar 2021 14:48:00 -0700 Subject: [PATCH 26/40] fix time mention in flag_uvf frequency check --- SSINS/incoherent_noise_spectrum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SSINS/incoherent_noise_spectrum.py b/SSINS/incoherent_noise_spectrum.py index 251ad3cb..fb959ca5 100644 --- a/SSINS/incoherent_noise_spectrum.py +++ b/SSINS/incoherent_noise_spectrum.py @@ -266,7 +266,7 @@ def flag_uvf(self, uvf, inplace=False): freq_compat = np.all(self.freq_array == test_freqs) assert freq_compat except Exception: - raise ValueError("UVFlag object's times do not match those of INS object.") + raise ValueError("UVFlag object's frequencies do not match those of INS object.") new_flags = self.mask_to_flags() if inplace: From 09d1b4ea8414212de369cbaa69a127734ae83bdb Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 26 Mar 2021 12:15:15 -0700 Subject: [PATCH 27/40] add freq_diff support to INS mask_to_flags() --- SSINS/incoherent_noise_spectrum.py | 18 +++++++++++------- SSINS/tests/test_INS.py | 4 ++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/SSINS/incoherent_noise_spectrum.py b/SSINS/incoherent_noise_spectrum.py index fb959ca5..679a5869 100644 --- a/SSINS/incoherent_noise_spectrum.py +++ b/SSINS/incoherent_noise_spectrum.py @@ -215,21 +215,25 @@ def mean_subtract(self, freq_slice=slice(None), return_coeffs=False): def mask_to_flags(self): """ Propagates the mask to construct flags for the original - (non time-differenced) data. If a time is flagged in the INS, then both + (non differenced) data. If a time is flagged in the INS, then both times that could have contributed to that time in the sky-subtraction step are flagged in the new array. Returns: - tp_flags (array): The time-propagated flags + p_flags (array): The (time or frequency)-propagated flags """ # Propagate the flags shape = list(self.metric_array.shape) - tp_flags = np.zeros([shape[0] + 1] + shape[1:], dtype=bool) - tp_flags[:-1] = self.metric_array.mask - tp_flags[1:] = np.logical_or(tp_flags[1:], tp_flags[:-1]) - - return(tp_flags) + if self.extra_keywords['dif_time'] is True: + p_flags = np.zeros([shape[0] + 1] + shape[1:], dtype=bool) + p_flags[:-1] = self.metric_array.mask + p_flags[1:] = np.logical_or(p_flags[1:], p_flags[:-1]) + if self.extra_keywords['dif_freq'] is True: + p_flags = np.zeros(shape[0] + [shape[1:] + 1], dtype=bool) + p_flags[:-1] = self.metric_array.mask + p_flags[1:] = np.logical_or(p_flags[1:], p_flags[:-1]) + return(p_flags) def flag_uvf(self, uvf, inplace=False): """ diff --git a/SSINS/tests/test_INS.py b/SSINS/tests/test_INS.py index a9aadf2d..e1a6fa39 100644 --- a/SSINS/tests/test_INS.py +++ b/SSINS/tests/test_INS.py @@ -255,6 +255,10 @@ def test_write_mwaf(): testfile = os.path.join(DATA_PATH, '%s.h5' % obs) prefix = os.path.join(DATA_PATH, '%s_test' % obs) ins = INS(testfile) + #to override the fact that the data files don't have dif_ keywords set + ins.extra_keywords['dif_time'] = True + ins.extra_keywords['dif_freq'] = False + mwaf_files = [os.path.join(DATA_PATH, '1061313128_12.mwaf')] bad_mwaf_files = [os.path.join(DATA_PATH, 'bad_file_path')] metafits_file = os.path.join(DATA_PATH, '1061313128.metafits') From e41a6930ea841d7298af722927d9c717400f1109 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 26 Mar 2021 12:17:20 -0700 Subject: [PATCH 28/40] use is to compare to singletons in INS flag_uvf() --- SSINS/incoherent_noise_spectrum.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SSINS/incoherent_noise_spectrum.py b/SSINS/incoherent_noise_spectrum.py index 679a5869..49ecf2d0 100644 --- a/SSINS/incoherent_noise_spectrum.py +++ b/SSINS/incoherent_noise_spectrum.py @@ -257,14 +257,14 @@ def flag_uvf(self, uvf, inplace=False): raise ValueError("UVFlag object must be in flag mode to write flags from INS object.") if uvf.type != 'waterfall': raise ValueError("UVFlag object must be in waterfall mode to write flags from INS object.") - if self.extra_keywords['dif_time'] == True: + if self.extra_keywords['dif_time'] is True: try: test_times = 0.5 * (uvf.time_array[:-1] + uvf.time_array[1:]) time_compat = np.all(self.time_array == test_times) assert time_compat except Exception: raise ValueError("UVFlag object's times do not match those of INS object.") - if self.extra_keywords['dif_freq'] == True: + if self.extra_keywords['dif_freq'] is True: try: test_freqs = 0.5 * (uvf.freq_array[:-1] + uvf.freq_array[1:]) freq_compat = np.all(self.freq_array == test_freqs) From 74f72eb82a6941b224c5f69b022a5763e50a626d Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 26 Mar 2021 13:54:09 -0700 Subject: [PATCH 29/40] have ss.read() apply_flags() in diff_freq --- SSINS/sky_subtract.py | 1 + 1 file changed, 1 insertion(+) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index 9dcaaf13..98be49d8 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -57,6 +57,7 @@ def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None self.extra_keywords['dif_freq'] = False if diff_freq: self.diff_freq() + self.apply_flags(flag_choice=flag_choice, INS=INS, custom=custom) self.extra_keywords['dif_freq'] = True self.extra_keywords['dif_time'] = False if not diff_freq and not diff: From a6987f09d4fafee5cae36263fc84623b59142c36 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 26 Mar 2021 13:55:22 -0700 Subject: [PATCH 30/40] properly arrange arrays for frequency slicing in INS.mask_to_flags --- SSINS/incoherent_noise_spectrum.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SSINS/incoherent_noise_spectrum.py b/SSINS/incoherent_noise_spectrum.py index 49ecf2d0..8431ee84 100644 --- a/SSINS/incoherent_noise_spectrum.py +++ b/SSINS/incoherent_noise_spectrum.py @@ -230,9 +230,9 @@ def mask_to_flags(self): p_flags[:-1] = self.metric_array.mask p_flags[1:] = np.logical_or(p_flags[1:], p_flags[:-1]) if self.extra_keywords['dif_freq'] is True: - p_flags = np.zeros(shape[0] + [shape[1:] + 1], dtype=bool) - p_flags[:-1] = self.metric_array.mask - p_flags[1:] = np.logical_or(p_flags[1:], p_flags[:-1]) + p_flags = np.zeros([shape[0], shape[1] + 1, shape[2]], dtype=bool) + p_flags[:,:-1] = self.metric_array.mask + p_flags[:,1:] = np.logical_or(p_flags[:,1:], p_flags[:,:-1]) return(p_flags) def flag_uvf(self, uvf, inplace=False): From 89106d509a3a63a2f818cb2228ec807a295f913d Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 26 Mar 2021 13:55:59 -0700 Subject: [PATCH 31/40] add test_mask_to_flags testo test_INS --- SSINS/tests/test_INS.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/SSINS/tests/test_INS.py b/SSINS/tests/test_INS.py index e1a6fa39..0c2da998 100644 --- a/SSINS/tests/test_INS.py +++ b/SSINS/tests/test_INS.py @@ -454,3 +454,13 @@ def test_add(): assert np.all(combo_ins.metric_array.mask == first_ins.metric_array.mask) assert np.all(combo_ins.metric_array.data == truth_ins.metric_array.data) assert np.all(combo_ins.metric_array.mask == truth_ins.metric_array.mask) + +def test_mask_to_flags(): + ss = SS() + filepath = 'SSINS/data/1061313128_99bl_1pol_half_time.uvfits' + ss.read(filepath, diff=True) + ins = INS(ss) + flags = ins.mask_to_flags() + ss.read(filepath, diff=False, diff_freq=True) + ins = INS(ss) + flags = ins.mask_to_flags() From 6fdab9990637ab744957fb78cab138f4c1725507 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Mon, 29 Mar 2021 13:34:59 -0700 Subject: [PATCH 32/40] add freq diff test for flag_uvf in INS --- SSINS/tests/test_INS.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/SSINS/tests/test_INS.py b/SSINS/tests/test_INS.py index 0c2da998..215e3ed8 100644 --- a/SSINS/tests/test_INS.py +++ b/SSINS/tests/test_INS.py @@ -128,6 +128,36 @@ def test_polyfit(): ins.metric_ms = ins.mean_subtract() assert np.all(ins.metric_ms.mask), "The metric_ms array was not all masked" +def test_flag_uvf_freq(): + obs = '1061313128_99bl_1pol_half_time' + testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) + file_type = 'uvfits' + prefix = os.path.join(DATA_PATH, '%s_test' % obs) + flags_outfile = '%s_SSINS_flags.h5' % prefix + + ss = SS() + ss.read(testfile, diff_freq=True) + + uvd = UVData() + uvd.read(testfile) + + uvf = UVFlag(uvd, mode='flag', waterfall=True) + # start with some flags so that we can test the intended OR operation + uvf.flag_array[6, :] = True + ins = INS(ss) + + # Check error handling + with pytest.raises(ValueError): + bad_uvf = UVFlag(uvd, mode='metric', waterfall=True) + err_uvf = ins.flag_uvf(uvf=bad_uvf) + with pytest.raises(ValueError): + bad_uvf = UVFlag(uvd, mode='flag', waterfall=False) + err_uvf = ins.flag_uvf(uvf=bad_uvf) + with pytest.raises(ValueError): + bad_uvf = UVFlag(uvd, mode='flag', waterfall=True) + # Pretend the data is off by 1 freq + bad_uvf.freq_array += 1 + err_uvf = ins.flag_uvf(uvf=bad_uvf) @pytest.mark.filterwarnings("ignore:Reordering", "ignore:SS.read") def test_mask_to_flags(): @@ -456,6 +486,7 @@ def test_add(): assert np.all(combo_ins.metric_array.mask == truth_ins.metric_array.mask) def test_mask_to_flags(): + #verify array sizes ss = SS() filepath = 'SSINS/data/1061313128_99bl_1pol_half_time.uvfits' ss.read(filepath, diff=True) From ebd737b3ee115c41d8417205e8d1f4d73ac9e70b Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 16 Apr 2021 12:39:05 -0700 Subject: [PATCH 33/40] comment flag propagation if/if statement --- SSINS/incoherent_noise_spectrum.py | 1 + 1 file changed, 1 insertion(+) diff --git a/SSINS/incoherent_noise_spectrum.py b/SSINS/incoherent_noise_spectrum.py index 8431ee84..afd3d728 100644 --- a/SSINS/incoherent_noise_spectrum.py +++ b/SSINS/incoherent_noise_spectrum.py @@ -225,6 +225,7 @@ def mask_to_flags(self): # Propagate the flags shape = list(self.metric_array.shape) + # no else here between dif/time, it is possible to have a doubly differenced set if self.extra_keywords['dif_time'] is True: p_flags = np.zeros([shape[0] + 1] + shape[1:], dtype=bool) p_flags[:-1] = self.metric_array.mask From a3a222cc111411626c54821450b9b06201ed1299 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 16 Apr 2021 12:39:53 -0700 Subject: [PATCH 34/40] warn on invalid extra_keywords entries --- SSINS/sky_subtract.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index 98be49d8..8171dfb2 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -47,6 +47,17 @@ def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None """ super().read(filename, **kwargs) + # detect if there is contridicting override_keyword and diff/diff_freq terms + if diff or diff_freq is True and override_keyword is not None: + warnings.warn("diff or diff_freq set to true and override keyword has" + " been set. Please ensure that this does not cause unwanted" + " behavior, since the override_keyword will override diff" + " setting internal attributes" + + if override_keyword != "dif_freq" or override_keyword != "dif_time" or override_keyword != "both": + warnings.warn("override_keyword passed but is not 'dif_time'," + " 'dif_freq', or 'both': override_keyword will have no" + " effect") # always set extra keywords, else key errors when trying to check if (self.data_array is not None): @@ -79,6 +90,7 @@ def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None self.extra_keywords['dif_freq'] = False # hardcoded cases for override_keyword to set dif_freq/dif_time (or both) # useful if data is pre-differenced rather than internally diff-ed by read() itself + # applied after the internal flags, and thus overwrites them if (override_keyword is not None): if override_keyword == 'dif_time': self.extra_keywords['dif_time'] = True From 689ff30724c58e3580817d5eb48e50e9088c4308 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 7 May 2021 12:16:20 -0700 Subject: [PATCH 35/40] fix missing closing paren on diff override warning --- SSINS/sky_subtract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SSINS/sky_subtract.py b/SSINS/sky_subtract.py index 8171dfb2..66b8832f 100644 --- a/SSINS/sky_subtract.py +++ b/SSINS/sky_subtract.py @@ -52,7 +52,7 @@ def read(self, filename, diff=False, diff_freq=False, flag_choice=None, INS=None warnings.warn("diff or diff_freq set to true and override keyword has" " been set. Please ensure that this does not cause unwanted" " behavior, since the override_keyword will override diff" - " setting internal attributes" + " setting internal attributes") if override_keyword != "dif_freq" or override_keyword != "dif_time" or override_keyword != "both": warnings.warn("override_keyword passed but is not 'dif_time'," From 96a3d698bf95891ba261fec3d7ab0ae437ffe234 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Thu, 11 Nov 2021 23:35:55 -0800 Subject: [PATCH 36/40] read data when testing masks --- SSINS/tests/test_SS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SSINS/tests/test_SS.py b/SSINS/tests/test_SS.py index eb746b0f..2b5ddafa 100644 --- a/SSINS/tests/test_SS.py +++ b/SSINS/tests/test_SS.py @@ -123,7 +123,7 @@ def test_diff_freq_mask(): ss = SS() #read in test file - ss.read(testfile, read_data=False, diff=False, diff_freq=False) + ss.read(testfile, read_data=True, diff=False, diff_freq=False) ss.apply_flags(flag_choice='original') assert ss.flag_array is not None temp_array = np.logical_or(ss.flag_array, ss.flag_array) From 432dad9fc6628f350c6b51e8c3073898478c4f51 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 3 Dec 2021 08:56:03 -0800 Subject: [PATCH 37/40] add masktoflags test but for freq --- SSINS/tests/test_INS.py | 72 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/SSINS/tests/test_INS.py b/SSINS/tests/test_INS.py index 215e3ed8..cdc71768 100644 --- a/SSINS/tests/test_INS.py +++ b/SSINS/tests/test_INS.py @@ -231,6 +231,78 @@ def test_mask_to_flags(): os.remove(flags_outfile) +@pytest.mark.filterwarnings("ignore:Reordering", "ignore:SS.read") +def test_mask_to_flags_freq(): + obs = '1061313128_99bl_1pol_half_time' + testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) + file_type = 'uvfits' + prefix = os.path.join(DATA_PATH, '%s_test' % obs) + flags_outfile = '%s_SSINS_flags.h5' % prefix + + ss = SS() + ss.read(testfile, diff_freq=True) + + uvd = UVData() + uvd.read(testfile) + + uvf = UVFlag(uvd, mode='flag', waterfall=True) + # start with some flags so that we can test the intended OR operation + uvf.flag_array[6, :] = True + ins = INS(ss) + + # Check error handling + with pytest.raises(ValueError): + bad_uvf = UVFlag(uvd, mode='metric', waterfall=True) + err_uvf = ins.flag_uvf(uvf=bad_uvf) + with pytest.raises(ValueError): + bad_uvf = UVFlag(uvd, mode='flag', waterfall=False) + err_uvf = ins.flag_uvf(uvf=bad_uvf) + with pytest.raises(ValueError): + bad_uvf = UVFlag(uvd, mode='flag', waterfall=True) + # Pretend the data is off by 1 day + bad_uvf.freq_array += 1 + err_uvf = ins.flag_uvf(uvf=bad_uvf) + + # Pretend we flagged the INS object + freq_inds_1 = np.arange(0, len(ins.freq_array), 2) + freq_inds_2 = np.arange(1, len(ins.freq_array), 2) + ins.metric_array[1, freq_inds_1] = np.ma.masked + ins.metric_array[3, freq_inds_1] = np.ma.masked + ins.metric_array[7, freq_inds_2] = np.ma.masked + ins.metric_array[-2, freq_inds_2] = np.ma.masked + + # Make a NEW uvflag object + new_uvf = ins.flag_uvf(uvf=uvf, inplace=False) + + # Construct the expected flags by hand + test_flags = np.zeros_like(new_uvf.flag_array) + test_flags[1:5, freq_inds_1] = True + test_flags[6, :] = True + test_flags[7, freq_inds_2] = True + test_flags[8, freq_inds_2] = True + test_flags[-3:-1, freq_inds_2] = True + + # Check that new flags are correct + assert np.all(new_uvf.flag_array == test_flags), "Test flags were not equal to calculated flags." + # Check that the input uvf was not edited in place + assert new_uvf != uvf, "The UVflag object was edited inplace and should not have been." + + # Edit the uvf inplace + inplace_uvf = ins.flag_uvf(uvf=uvf, inplace=True) + + # Check that new flags are correct + assert np.all(inplace_uvf.flag_array == test_flags), "Test flags were not equal to calculated flags." + # Check that the input uvf was not edited in place + assert inplace_uvf == uvf, "The UVflag object was not edited inplace and should have been." + + # Test write/read + ins.write(prefix, output_type='flags', uvf=uvf) + read_uvf = UVFlag(flags_outfile, mode='flag', waterfall=True) + # Check equality + assert read_uvf == uvf, "UVFlag objsect differs after read" + + os.remove(flags_outfile) + @pytest.mark.filterwarnings("ignore:Reordering", "ignore:SS.read", "ignore:invalid value") def test_write(): From 7f4a7a573c4b0a2135e6fd19efa8793e5e244032 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 3 Dec 2021 09:04:38 -0800 Subject: [PATCH 38/40] Revert "add masktoflags test but for freq" This reverts commit 432dad9fc6628f350c6b51e8c3073898478c4f51. --- SSINS/tests/test_INS.py | 72 ----------------------------------------- 1 file changed, 72 deletions(-) diff --git a/SSINS/tests/test_INS.py b/SSINS/tests/test_INS.py index cdc71768..215e3ed8 100644 --- a/SSINS/tests/test_INS.py +++ b/SSINS/tests/test_INS.py @@ -231,78 +231,6 @@ def test_mask_to_flags(): os.remove(flags_outfile) -@pytest.mark.filterwarnings("ignore:Reordering", "ignore:SS.read") -def test_mask_to_flags_freq(): - obs = '1061313128_99bl_1pol_half_time' - testfile = os.path.join(DATA_PATH, '%s.uvfits' % obs) - file_type = 'uvfits' - prefix = os.path.join(DATA_PATH, '%s_test' % obs) - flags_outfile = '%s_SSINS_flags.h5' % prefix - - ss = SS() - ss.read(testfile, diff_freq=True) - - uvd = UVData() - uvd.read(testfile) - - uvf = UVFlag(uvd, mode='flag', waterfall=True) - # start with some flags so that we can test the intended OR operation - uvf.flag_array[6, :] = True - ins = INS(ss) - - # Check error handling - with pytest.raises(ValueError): - bad_uvf = UVFlag(uvd, mode='metric', waterfall=True) - err_uvf = ins.flag_uvf(uvf=bad_uvf) - with pytest.raises(ValueError): - bad_uvf = UVFlag(uvd, mode='flag', waterfall=False) - err_uvf = ins.flag_uvf(uvf=bad_uvf) - with pytest.raises(ValueError): - bad_uvf = UVFlag(uvd, mode='flag', waterfall=True) - # Pretend the data is off by 1 day - bad_uvf.freq_array += 1 - err_uvf = ins.flag_uvf(uvf=bad_uvf) - - # Pretend we flagged the INS object - freq_inds_1 = np.arange(0, len(ins.freq_array), 2) - freq_inds_2 = np.arange(1, len(ins.freq_array), 2) - ins.metric_array[1, freq_inds_1] = np.ma.masked - ins.metric_array[3, freq_inds_1] = np.ma.masked - ins.metric_array[7, freq_inds_2] = np.ma.masked - ins.metric_array[-2, freq_inds_2] = np.ma.masked - - # Make a NEW uvflag object - new_uvf = ins.flag_uvf(uvf=uvf, inplace=False) - - # Construct the expected flags by hand - test_flags = np.zeros_like(new_uvf.flag_array) - test_flags[1:5, freq_inds_1] = True - test_flags[6, :] = True - test_flags[7, freq_inds_2] = True - test_flags[8, freq_inds_2] = True - test_flags[-3:-1, freq_inds_2] = True - - # Check that new flags are correct - assert np.all(new_uvf.flag_array == test_flags), "Test flags were not equal to calculated flags." - # Check that the input uvf was not edited in place - assert new_uvf != uvf, "The UVflag object was edited inplace and should not have been." - - # Edit the uvf inplace - inplace_uvf = ins.flag_uvf(uvf=uvf, inplace=True) - - # Check that new flags are correct - assert np.all(inplace_uvf.flag_array == test_flags), "Test flags were not equal to calculated flags." - # Check that the input uvf was not edited in place - assert inplace_uvf == uvf, "The UVflag object was not edited inplace and should have been." - - # Test write/read - ins.write(prefix, output_type='flags', uvf=uvf) - read_uvf = UVFlag(flags_outfile, mode='flag', waterfall=True) - # Check equality - assert read_uvf == uvf, "UVFlag objsect differs after read" - - os.remove(flags_outfile) - @pytest.mark.filterwarnings("ignore:Reordering", "ignore:SS.read", "ignore:invalid value") def test_write(): From 5169e784e82eb8fc25b1a17735e5244a20174571 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Fri, 7 Jan 2022 09:37:43 -0800 Subject: [PATCH 39/40] trivially rename test_mask_to_flags second test to avoid collision --- SSINS/tests/test_INS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SSINS/tests/test_INS.py b/SSINS/tests/test_INS.py index 215e3ed8..f3a84f4b 100644 --- a/SSINS/tests/test_INS.py +++ b/SSINS/tests/test_INS.py @@ -485,7 +485,7 @@ def test_add(): assert np.all(combo_ins.metric_array.data == truth_ins.metric_array.data) assert np.all(combo_ins.metric_array.mask == truth_ins.metric_array.mask) -def test_mask_to_flags(): +def test_mask_to_flags_2(): #verify array sizes ss = SS() filepath = 'SSINS/data/1061313128_99bl_1pol_half_time.uvfits' From 1cfb8206f65fb31082485f31cbd42f4e7efe1aa8 Mon Sep 17 00:00:00 2001 From: Alex Foster <123afoster@gmail.com> Date: Tue, 11 Jan 2022 14:10:03 -0800 Subject: [PATCH 40/40] add extra mask_to_flags_2 test for diff freq+time --- SSINS/tests/test_INS.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/SSINS/tests/test_INS.py b/SSINS/tests/test_INS.py index f3a84f4b..027d7f6d 100644 --- a/SSINS/tests/test_INS.py +++ b/SSINS/tests/test_INS.py @@ -495,3 +495,6 @@ def test_mask_to_flags_2(): ss.read(filepath, diff=False, diff_freq=True) ins = INS(ss) flags = ins.mask_to_flags() + ss.read(filepath, diff=True, diff_freq=True) + ins = INS(ss) + flags = ins.mask_to_flags()