Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix flagging logic for missing and unknown data in QARTOD spike test #129

Merged
merged 2 commits into from
Jan 30, 2025
Merged

Fix flagging logic for missing and unknown data in QARTOD spike test #129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
070a3d5
Fix flagging logic for missing and unknown data in QARTOD spike test
Sakshamgupta90 Aug 13, 2024
421ee5d
address @iwensu0313 review comments
ocefpaf Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions ioos_qc/qartod.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -612,8 +612,19 @@ def spike_test(
flag_arr[0] = QartodFlags.UNKNOWN
flag_arr[-1] = QartodFlags.UNKNOWN

# If the value is masked or nan set the flag to MISSING
flag_arr[diff.mask] = QartodFlags.MISSING
# Check if the original data was masked
for i in range(inp.size):

# Check if inp is masked (original data missing)
if inp.mask[i]:
flag_arr[i] = QartodFlags.MISSING

# Check if diff is masked but not in inp (this indicates that original data is not missing,
# but the data point got masked in diff lines 575-580 due to trying to calculate a value
# using a valid value and a missing value; and because of that, we are not able to apply QARTOD
# thus the UNKNOWN flag)
elif (diff.mask[i] and not inp.mask[i]):
flag_arr[i] = QartodFlags.UNKNOWN

return flag_arr.reshape(original_shape)

Expand Down
86 changes: 5 additions & 81 deletions tests/test_qartod.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1018,7 +1018,7 @@ def test_spike_masked(self):

# First and last elements should always be good data, unless someone
# has set a threshold to zero.
expected = [2, 4, 4, 4, 1, 3, 1, 9, 9, 9, 4, 4, 9, 9]
expected = [2, 4, 4, 4, 1, 3, 1, 2, 9, 2, 4, 4, 2, 9]

inputs = [
arr,
Expand Down Expand Up @@ -1126,46 +1126,8 @@ def test_spike_methods(self):
inp = [3, 4.99, 5, 6, 8, 6, 6, 6.75, 6, 6, 5.3, 6, 6, 9, 5, None, 4, 4]
suspect_threshold = 0.5
fail_threshold = 1
average_method_expected = [
2,
3,
1,
1,
4,
3,
1,
3,
1,
1,
3,
1,
4,
4,
9,
9,
9,
2,
]
diff_method_expected = [
2,
1,
1,
1,
4,
1,
1,
3,
1,
1,
3,
1,
1,
4,
9,
9,
9,
2,
]
average_method_expected = [2, 3, 1, 1, 4, 3, 1, 3, 1, 1, 3, 1, 4, 4, 2, 9, 2, 2]
diff_method_expected = [2, 1, 1, 1, 4, 1, 1, 3, 1, 1, 3, 1, 1, 4, 2, 9, 2, 2]

# Test average method
npt.assert_array_equal(
Expand Down Expand Up @@ -1222,46 +1184,8 @@ def test_spike_test_bad_method(self):

def test_spike_test_inputs(self):
inp = [3, 4.99, 5, 6, 8, 6, 6, 6.75, 6, 6, 5.3, 6, 6, 9, 5, None, 4, 4]
expected_suspect_only = [
2,
3,
1,
1,
3,
3,
1,
3,
1,
1,
3,
1,
3,
3,
9,
9,
9,
2,
]
expected_fail_only = [
2,
1,
1,
1,
4,
1,
1,
1,
1,
1,
1,
1,
4,
4,
9,
9,
9,
2,
]
expected_suspect_only = [2, 3, 1, 1, 3, 3, 1, 3, 1, 1, 3, 1, 3, 3, 2, 9, 2, 2]
expected_fail_only = [2, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, 4, 4, 2, 9, 2, 2]
suspect_threshold = 0.5
fail_threshold = 1

Expand Down