diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index 7ccc132..d4244d9 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -638,12 +638,16 @@ def rate_of_change_test( inp: Sequence[Real], tinp: Sequence[Real], threshold: float, + fail_threshold: float | None = None, ) -> np.ma.core.MaskedArray: """Checks the first order difference of a series of values to see if there are any values exceeding a threshold defined by the inputs. These are then marked as SUSPECT. It is up to the test operator to determine an appropriate threshold value for the absolute difference not to exceed. Threshold is expressed as a rate in observations units per second. + There is an optional fail_threshold parameter that can be set by the user. + If the first order difference of a series of values exceeds this higher threshold, + the data point is marked as FAIL. Missing and masked data is flagged as UNKNOWN. Parameters @@ -657,6 +661,10 @@ def rate_of_change_test( If anything else is passed in the format is assumed to be seconds since the unix epoch. threshold A float value representing a rate of change over time, in observation units per second. + Rates exceeding this will be flagged as SUSPECT. + fail_threshold + A float value representing a rate of change over time, in observation units per second. + Rates exceeding this will be flagged as FAIL. Returns ------- @@ -686,6 +694,11 @@ def rate_of_change_test( with np.errstate(invalid="ignore"): flag_arr[roc > threshold] = QartodFlags.SUSPECT + # the fail threshold is optional and only required for some use cases + if fail_threshold is not None: + with np.errstate(invalid="ignore"): + flag_arr[roc > fail_threshold] = QartodFlags.FAIL + # If the value is masked set the flag to MISSING flag_arr[inp.mask] = QartodFlags.MISSING diff --git a/tests/test_qartod.py b/tests/test_qartod.py index 2f035b5..63d22a7 100644 --- a/tests/test_qartod.py +++ b/tests/test_qartod.py @@ -1228,6 +1228,7 @@ def setUp(self): ) self.times_epoch_secs = [t.astype(int) for t in self.times] self.threshold = 5 / 15 / 60 # 5 units per 15 minutes --> 5/15/60 units per second + self.fail_threshold = 10 / 15 / 60 # 10 units per 15 minutes --> 10/15/60 units per second def test_rate_of_change(self): times = self.times @@ -1328,6 +1329,86 @@ def test_rate_of_change_negative_values(self): ) npt.assert_array_equal(expected, result) + def test_rate_of_change_fail_flag(self): + """Test of optional fail flag.""" + times = self.times + arr = [ + 2, + 10, + 2.1, + 3, + 4, + 5, + 7, + 10, + 0, + 2, + 2.2, + 2, + 1, + 2, + 3, + 90, + 91, + 92, + 93, + 1, + 2, + 3, + 4, + 5, + ] + expected = [ + 1, + 3, + 3, + 1, + 1, + 1, + 1, + 1, + 4, + 1, + 1, + 1, + 1, + 1, + 1, + 4, + 1, + 1, + 1, + 4, + 1, + 1, + 1, + 1, + ] + inputs = [ + arr, + np.asarray(arr, dtype=np.float64), + dask_arr(np.asarray(arr, dtype=np.float64)), + ] + for i in inputs: + result = qartod.rate_of_change_test( + inp=i, + tinp=times, + threshold=self.threshold, + fail_threshold=self.fail_threshold, + ) + npt.assert_array_equal(expected, result) + + # test epoch secs - should return same result + npt.assert_array_equal( + qartod.rate_of_change_test( + inp=i, + tinp=self.times_epoch_secs, + threshold=self.threshold, + fail_threshold=self.fail_threshold, + ), + expected, + ) + class QartodFlatLineTest(unittest.TestCase): def setUp(self):