From a96c59f490eb7e27fed9abbe688967bcb4b94041 Mon Sep 17 00:00:00 2001 From: Jessy Barrette Date: Tue, 15 Dec 2020 16:08:37 -0800 Subject: [PATCH 01/10] Initial commit of range from target feature of the location_test --- ioos_qc/qartod.py | 28 +++++++++++++++++++++------- ioos_qc/utils.py | 6 ++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index 5ed9af4..7cd9c85 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -15,10 +15,10 @@ isfixedlength, add_flag_metadata, great_circle_distance, - mapdates + mapdates, + distance_from_target ) - L = logging.getLogger(__name__) # noqa @@ -85,10 +85,13 @@ def qartod_compare(vectors : Sequence[Sequence[N]] @add_flag_metadata(standard_name='location_test_quality_flag', long_name='Location Test Quality Flag') -def location_test(lon : Sequence[N], - lat : Sequence[N], - bbox : Tuple[N, N, N, N] = (-180, -90, 180, 90), - range_max : N = None +def location_test(lon: Sequence[N], + lat: Sequence[N], + bbox: Tuple[N, N, N, N] = (-180, -90, 180, 90), + range_max: N = None, + target_lat: N = None, + target_lon: N = None, + target_range: N = None ) -> np.ma.core.MaskedArray: """Checks that a location is within reasonable bounds. @@ -147,7 +150,18 @@ def location_test(lon : Sequence[N], d = great_circle_distance(lat, lon) flag_arr[d > range_max] = QartodFlags.SUSPECT - # Ignore warnings when comparing NaN values even though they are masked + # Distance From Target Test + if target_lat is not None and target_lon is not None and \ + target_range is not None: + if len(target_lon) == 1 and len(target_lat) == 1: + # If one target lat/lon and range is given get the distance + # from the target + d_from_target = distance_from_target(lat, lon, + target_lat * np.ones(lat.size), target_lon * np.ones(lat.size)) + + flag_arr[d_from_target > target_range] = QartodFlags.SUSPECT + + # Ignore warnings when comparing NaN values even though they are masked # https://github.com/numpy/numpy/blob/master/doc/release/1.8.0-notes.rst#runtime-warnings-when-comparing-nan-numbers with np.errstate(invalid='ignore'): flag_arr[(lon < bbox.minx) | (lat < bbox.miny) | diff --git a/ioos_qc/utils.py b/ioos_qc/utils.py index 417ef9e..ea9b854 100644 --- a/ioos_qc/utils.py +++ b/ioos_qc/utils.py @@ -263,3 +263,9 @@ def great_circle_distance(lat_arr, lon_arr): g = Geod(ellps='WGS84') _, _, dist[1:] = g.inv(lon_arr[:-1], lat_arr[:-1], lon_arr[1:], lat_arr[1:]) return dist + + +def distance_from_target(lat, lon, target_lat, target_lon): + g = Geod(ellps='WGS84') + _, _, dist_to_target = g.inv(lon, lat, target_lon, target_lat) + return dist_to_target From d35443455964c898fdb7b1b944a4ef15681df0c6 Mon Sep 17 00:00:00 2001 From: Jessy Barrette Date: Wed, 27 Jan 2021 15:14:21 -0800 Subject: [PATCH 02/10] Handle array input of target position the same size of lat/lon --- ioos_qc/qartod.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index 516c1d5..861b95f 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -89,8 +89,8 @@ def location_test(lon: Sequence[N], lat: Sequence[N], bbox: Tuple[N, N, N, N] = (-180, -90, 180, 90), range_max: N = None, - target_lat: N = None, - target_lon: N = None, + target_lat: [N] = None, + target_lon: [N] = None, target_range: N = None ) -> np.ma.core.MaskedArray: """Checks that a location is within reasonable bounds. @@ -154,11 +154,13 @@ def location_test(lon: Sequence[N], if target_lat is not None and target_lon is not None and \ target_range is not None: if len(target_lon) == 1 and len(target_lat) == 1: - # If one target lat/lon and range is given get the distance - # from the target + # If only one value is given assume to be constant for all positions d_from_target = distance_from_target(lat, lon, target_lat * np.ones(lat.size), target_lon * np.ones(lat.size)) + elif target_lon.shape == lon.shape and target_lat == lon.shape: + d_from_target = distance_from_target(lat, lon, target_lat, target_lon) + # Flag as suspect distances greater than target_range flag_arr[d_from_target > target_range] = QartodFlags.SUSPECT # Ignore warnings when comparing NaN values even though they are masked From e3ef97ad00efa78b62cd8c1ea8e38520b9cccdcb Mon Sep 17 00:00:00 2001 From: Jessy Barrette Date: Wed, 27 Jan 2021 15:15:31 -0800 Subject: [PATCH 03/10] Add a description of the target inputs within the test description --- ioos_qc/qartod.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index 861b95f..cf6262a 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -106,6 +106,11 @@ def location_test(lon: Sequence[N], lat: Latitudes as a numeric numpy array or a list of numbers. bbox: A length 4 tuple expressed in (minx, miny, maxx, maxy) [optional]. range_max: Maximum allowed range expressed in geodesic curve distance (meters). + target_lat: Target Latitude as numeric numpy array or a list of numbers, + it can either same size as lat/lon or a unique values + target_lon: Target Longitude as numeric numpy array or a list of numbers, + it can either same size as lat/lon or a unique values + target_range: Maximum allowed range in geodesic curve distance (meters) away from target position. Returns: A masked array of flag values equal in size to that of the input. From 4a11d1c90676e60e14eb4734a09c7a98012d22c3 Mon Sep 17 00:00:00 2001 From: Jessy Barrette Date: Wed, 27 Jan 2021 17:01:14 -0800 Subject: [PATCH 04/10] Regroup to the target single and multiple inputs --- ioos_qc/qartod.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index cf6262a..0760039 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -158,12 +158,12 @@ def location_test(lon: Sequence[N], # Distance From Target Test if target_lat is not None and target_lon is not None and \ target_range is not None: - if len(target_lon) == 1 and len(target_lat) == 1: - # If only one value is given assume to be constant for all positions - d_from_target = distance_from_target(lat, lon, - target_lat * np.ones(lat.size), target_lon * np.ones(lat.size)) - elif target_lon.shape == lon.shape and target_lat == lon.shape: - d_from_target = distance_from_target(lat, lon, target_lat, target_lon) + # If only one value is given assume to be constant for all positions + if target_lon.size == 1 and target_lat.size == 1: + (target_lon, target_lat) = (target_lon * np.ones(lat.size), target_lat * np.ones(lat.size)) + + # Compute the range from the target location + d_from_target = distance_from_target(lat, lon, target_lat, target_lon) # Flag as suspect distances greater than target_range flag_arr[d_from_target > target_range] = QartodFlags.SUSPECT From 18c44bf35cb430823b5438069dda20db7249a836 Mon Sep 17 00:00:00 2001 From: Jessy Barrette Date: Wed, 27 Jan 2021 17:01:52 -0800 Subject: [PATCH 05/10] Add a missing flag if one target location is missing in an array --- ioos_qc/qartod.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index 0760039..ebe7e4f 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -168,6 +168,9 @@ def location_test(lon: Sequence[N], # Flag as suspect distances greater than target_range flag_arr[d_from_target > target_range] = QartodFlags.SUSPECT + # Flag as missing target_location distance + flag_arr[(target_lat.mask | target_lon.mask)] = QartodFlags.MISSING + # Ignore warnings when comparing NaN values even though they are masked # https://github.com/numpy/numpy/blob/master/doc/release/1.8.0-notes.rst#runtime-warnings-when-comparing-nan-numbers with np.errstate(invalid='ignore'): From b9c97a332681c1be2184602d5293c51837a8d602 Mon Sep 17 00:00:00 2001 From: Jessy Barrette Date: Wed, 27 Jan 2021 17:02:25 -0800 Subject: [PATCH 06/10] Add a few checks for bad target inputs --- ioos_qc/qartod.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index ebe7e4f..a2c8143 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -138,6 +138,29 @@ def location_test(lon: Sequence[N], lon = lon.flatten() lat = lat.flatten() + # Handle target inputs + if target_lon is not None or target_lat is not None or target_range is not None: + if target_lon is not None and target_lat is not None and target_range is not None: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + target_lat = np.ma.masked_invalid(np.array(target_lat).astype(np.float64)) + target_lon = np.ma.masked_invalid(np.array(target_lon).astype(np.float64)) + elif target_lon is not None and target_lat is not None and target_range is None: + raise ValueError('Missing target_range input if target_lat and target_lon are provided') + else: + raise ValueError('Missing some target inputs') + + if target_lon.shape != target_lat.shape: + raise ValueError( + 'Target_lon ({0.shape}) and target_lat ({1.shape}) are different shapes'.format( + target_lon, target_lat + ) + ) + + # Flatten target_lon and target_lat + target_lon = target_lon.flatten() + target_lat = target_lat.flatten() + # Start with everything as passing (1) flag_arr = np.ma.ones(lon.size, dtype='uint8') From c1e08421c0ceb2cb17743d03f0af9a2d656c28ef Mon Sep 17 00:00:00 2001 From: Jessy Barrette Date: Thu, 28 Jan 2021 09:27:21 -0800 Subject: [PATCH 07/10] distance_from_target should output a masked array --- ioos_qc/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ioos_qc/utils.py b/ioos_qc/utils.py index ea9b854..b8d0ee3 100644 --- a/ioos_qc/utils.py +++ b/ioos_qc/utils.py @@ -268,4 +268,5 @@ def great_circle_distance(lat_arr, lon_arr): def distance_from_target(lat, lon, target_lat, target_lon): g = Geod(ellps='WGS84') _, _, dist_to_target = g.inv(lon, lat, target_lon, target_lat) + dist_to_target = np.ma.masked_invalid(dist_to_target.astype(np.float64)) return dist_to_target From adbb3cbfd5c132470d4db5f5c431193666f6e971 Mon Sep 17 00:00:00 2001 From: Jessy Barrette Date: Thu, 28 Jan 2021 09:28:10 -0800 Subject: [PATCH 08/10] Ignore warnings when applying Suspect flag with range from target --- ioos_qc/qartod.py | 3 ++- tests/test_qartod.py | 56 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index a2c8143..29895dd 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -189,7 +189,8 @@ def location_test(lon: Sequence[N], d_from_target = distance_from_target(lat, lon, target_lat, target_lon) # Flag as suspect distances greater than target_range - flag_arr[d_from_target > target_range] = QartodFlags.SUSPECT + with np.errstate(invalid='ignore'): + flag_arr[d_from_target > target_range] = QartodFlags.SUSPECT # Flag as missing target_location distance flag_arr[(target_lat.mask | target_lon.mask)] = QartodFlags.MISSING diff --git a/tests/test_qartod.py b/tests/test_qartod.py index 8c89b22..9b9176d 100644 --- a/tests/test_qartod.py +++ b/tests/test_qartod.py @@ -155,6 +155,62 @@ def test_location_distance_threshold(self): np.ma.array([1, 1, 3]) ) + def test_location_single_target_threshold(self): + lon = np.array([-71.05, -71.06, -80.0]) + lat = np.array([41.0, 41.02, 45.05]) + + npt.assert_array_equal( + qartod.location_test(lon, lat, target_range=3000.0, target_lon=-71.06, target_lat=41), + np.ma.array([1, 1, 3]) + ) + + def test_location_multiple_target_threshold(self): + lon = np.array([-71.05, -71.06, -80.0]) + lat = np.array([41.0, 41.02, 45.05]) + target_lon = np.array([-71.05, -75.06, -80.0]) + target_lat = np.array([41.0, 41.02, 45.05]) + + npt.assert_array_equal( + qartod.location_test(lon, lat, target_range=3000.0, target_lon=target_lon, target_lat=target_lat), + np.ma.array([1, 3, 1]) + ) + + def test_location_multiple_target_missing_target(self): + lon = np.array([-71.05, -71.06, -80.0, -80.0]) + lat = np.array([41.0, 41.02, 45.05, 45.05]) + target_lon = np.array([-71.05, None, -80.0, None]) + target_lat = np.array([41.0, 41.02, None, None]) + + npt.assert_array_equal( + qartod.location_test(lon, lat, target_range=3000.0, target_lon=target_lon, target_lat=target_lat), + np.ma.array([1, 9, 9, 9]) + ) + + def test_location_target_missing_threshold(self): + lon = np.array([-71.05, -71.06, -80.0]) + lat = np.array([41.0, 41.02, 45.05]) + target_lon = np.array([-71.05, -75.06, -80.0]) + target_lat = np.array([41.0, 41.02, 45.05]) + + with self.assertRaises(ValueError): + qartod.location_test(lon, lat, target_range=None, target_lon=target_lon, target_lat=target_lat) + with self.assertRaises(ValueError): + qartod.location_test(lon, lat, target_range=None, target_lon=target_lon[0], target_lat=target_lat[0]) + + def test_location_target_missing_threshold(self): + lon = np.array([-71.05, -71.06, -80.0]) + lat = np.array([41.0, 41.02, 45.05]) + target_lon = np.array([-71.05, -75.06, -80.0]) + target_lat = np.array([41.0, 41.02, 45.05]) + + with self.assertRaises(ValueError): + qartod.location_test(lon, lat, target_range=3000, target_lat=target_lat) + with self.assertRaises(ValueError): + qartod.location_test(lon, lat, target_range=None, target_lon=target_lon) + with self.assertRaises(ValueError): + qartod.location_test(lon, lat, target_range=3000, target_lat=45) + with self.assertRaises(ValueError): + qartod.location_test(lon, lat, target_range=3000, target_lon=45) class QartodGrossRangeTest(unittest.TestCase): From 776216efba75a79f07e95704e072059c58cb96b4 Mon Sep 17 00:00:00 2001 From: Jessy Barrette Date: Thu, 28 Jan 2021 11:12:08 -0800 Subject: [PATCH 09/10] Add a check to make sure that target_range is either float or int --- ioos_qc/qartod.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index 29895dd..8278be9 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -139,12 +139,17 @@ def location_test(lon: Sequence[N], lat = lat.flatten() # Handle target inputs + # If any target inputs are provided if target_lon is not None or target_lat is not None or target_range is not None: + # All target inputs should be there if target_lon is not None and target_lat is not None and target_range is not None: with warnings.catch_warnings(): warnings.simplefilter("ignore") target_lat = np.ma.masked_invalid(np.array(target_lat).astype(np.float64)) target_lon = np.ma.masked_invalid(np.array(target_lon).astype(np.float64)) + if type(target_range) not in [int, float]: + raise ValueError('Bad target_range input. target_range should either float or int.') + elif target_lon is not None and target_lat is not None and target_range is None: raise ValueError('Missing target_range input if target_lat and target_lon are provided') else: From a39ff757f701a9571e79434cd0445d6d31ad6088 Mon Sep 17 00:00:00 2001 From: Jessy Barrette Date: Thu, 28 Jan 2021 11:12:33 -0800 Subject: [PATCH 10/10] Add a few more tests to location_test --- tests/test_qartod.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_qartod.py b/tests/test_qartod.py index 9b9176d..3273c68 100644 --- a/tests/test_qartod.py +++ b/tests/test_qartod.py @@ -117,6 +117,18 @@ def test_location_bad_input(self): with self.assertRaises(ValueError): qartod.location_test(lon=70, lat=70, bbox=(1, 2)) + # Wrong target lon + with self.assertRaises(ValueError): + qartod.location_test(lon=70, lat=70, bbox=(1, 2, 3, 4), target_lon='foo', target_lat=70, target_range=3000) + + # Wrong target lat + with self.assertRaises(ValueError): + qartod.location_test(lon=70, lat=70, bbox=(1, 2, 3, 4), target_lon=70, target_lat='bad', target_range=3000) + + # Wrong target range + with self.assertRaises(ValueError): + qartod.location_test(lon=70, lat=70, bbox=(1, 2, 3, 4), target_lon=70, target_lat=70, target_range='300') + def test_location_bbox(self): lon = [80, -78, -71, -79, 500] lat = [None, 50, 59, 10, -60] @@ -212,6 +224,18 @@ def test_location_target_missing_threshold(self): with self.assertRaises(ValueError): qartod.location_test(lon, lat, target_range=3000, target_lon=45) + def test_location_dual_threshold_test(self): + lon = np.array([-71.05, -71.06, -80.0, -80.0]) + lat = np.array([41.0, 41.02, 45.05, 45.05]) + target_lon = np.array([-71.05, -71.06, -80.0, -80.0]) + target_lat = np.array([41.0, 41.02, 45.05, 46.05]) + + npt.assert_array_equal( + qartod.location_test(lon, lat, range_max=3000, + target_range=3000.0, target_lon=target_lon, target_lat=target_lat), + np.ma.array([1, 1, 3, 3]) + ) + class QartodGrossRangeTest(unittest.TestCase): def test_gross_range_check(self):