JP-3967: SATURATION flag for partially saturated pixels should be propagated to the _rate DQ

Author: pllim

changes/421.apichange.rst

@@ -0,0 +1 @@
+Set saturation flag for any saturation. This reverts back to saturation flagging behavior prior to #125

docs/stcal/ramp_fitting/description.rst

@@ -195,11 +195,11 @@ Segment-specific Computations
 +++++++++++++++++++++++++++++
 The variance of the slope of a segment due to read noise is:
 
-.. math::  
+.. math::
    var^R_{s} = \frac{12 \ R^2 }{ (ngroups_{s}^3 - ngroups_{s})(tgroup^2)(gain^2) } \,,
 
-where :math:`R` is the noise in the difference between 2 frames, 
-:math:`ngroups_{s}` is the number of groups in the segment, and :math:`tgroup` is the group 
+where :math:`R` is the noise in the difference between 2 frames,
+:math:`ngroups_{s}` is the number of groups in the segment, and :math:`tgroup` is the group
 time in seconds (from the keyword TGROUP).  The divide by gain converts to
 :math:`DN`.  For the special case where as segment has length 1, the
 :math:`ngroups_{s}` is set to :math:`2`.
@@ -272,7 +272,7 @@ The square-root of the combined variance is stored in the ERR array of the outpu
 The overall slope depends on the slope and the combined variance of the slope of each integration's
 segments, so is a sum over integration values computed from the segements:
 
-.. math::    
+.. math::
     slope_{o} = \frac{ \sum_{i}{ \frac{slope_{i}} {var^C_{i}}}} { \sum_{i}{ \frac{1} {var^C_{i}}}}
 
 
@@ -300,7 +300,7 @@ Data Quality Propagation
 For a given pixel, if all groups in an integration are flagged as DO_NOT_USE or
 SATURATED, then that pixel will be flagged as DO_NOT_USE in the corresponding
 integration in the "rateints" product.  Note this does NOT mean that all groups
-are flagged as SATURATED, nor that all groups are flagged as DO_NOT_USE.  For
+are flagged as DO_NOT_USE.  For
 example, slope calculations that are suppressed due to a ramp containing only
 one good group will be flagged as DO_NOT_USE in the
 first group, but not necessarily any other group, while only groups two and
@@ -312,9 +312,9 @@ For a given pixel, if all groups in an integration are flagged as SATURATED,
 then that pixel will be flagged as SATURATED and DO_NOT_USE in the corresponding
 integration in the "rateints" product.  This is different from the above case in
 that this is only for all groups flagged as SATURATED, not for some combination
-of DO_NOT_USE and SATURATED.  Further, only if all integrations in the "rateints"
-product are flagged as SATURATED, then the pixel will be flagged as SATURATED
-and DO_NOT_USE in the "rate" product.
+of DO_NOT_USE and SATURATED.  Furthermore, if any integration in the "rateints"
+product is flagged as SATURATED, then the pixel will be flagged as SATURATED
+in the "rate" product.
 
 For a given pixel, if any group in an integration is flagged as JUMP_DET, then
 that pixel will be flagged as JUMP_DET in the corresponding integration in the
@@ -338,12 +338,12 @@ group/resultant directly, the likelihood algorithm is based on differences of
 the groups/resultants :math:`d_i = r_i - r_{i-1}`.  The model used to determine
 the slope/countrate, :math:`a`, is:
 
-.. math::    
+.. math::
     \chi^2 = ({\bf d} - a \cdot {\bf 1})^T C ({\bf d} - a \cdot {\bf 1}) \,,
 
-Differentiating, setting to zero, then solving for :math:`a` results in 
+Differentiating, setting to zero, then solving for :math:`a` results in
 
-.. math::    
+.. math::
     a = ({\bf 1}^T C {\bf d})({\bf 1}^T C {\bf 1})^T \,,
 
 The covariance matrix :math:`C` is a tridiagonal matrix, due to the nature of the

src/stcal/ramp_fitting/utils.py

@@ -2,7 +2,6 @@
 #
 # utils.py: utility functions
 import logging
-import warnings
 
 import numpy as np
 
@@ -146,11 +145,11 @@ def dq_compress_final(dq_int, ramp_data):
     dnu = np.uint32(ramp_data.flags_do_not_use)
     sat = np.uint32(ramp_data.flags_saturated)
 
-    # Remove DO_NOT_USE and SATURATED because they need special handling.
-    # These flags are not set in the final pixel DQ array by simply being set
+    # Remove DO_NOT_USE because it needs special handling.
+    # This flag is not set in the final pixel DQ array by simply being set
     # in one of the integrations.
-    not_sat_or_dnu = np.uint32(~(dnu | sat))
-    final_dq = np.bitwise_and(final_dq, not_sat_or_dnu)
+    not_dnu = np.uint32(~dnu)
+    final_dq = np.bitwise_and(final_dq, not_dnu)
 
     # If all integrations are DO_NOT_USE or SATURATED, then set DO_NOT_USE.
     set_if_total_integ(final_dq, dq_int, dnu | sat, dnu)

tests/test_ramp_fitting.py

@@ -1,4 +1,3 @@
-import pytest
 import numpy as np
 
 import sys
@@ -449,7 +448,6 @@ def test_2_group_cases():
     """
     # XXX JP-3121: Still needs work
     base_group = [-12328.601, -4289.051]
-    base_err = [0.0, 0.0]
     gain_val = 0.9699
     rnoise_val = 9.4552
 
@@ -511,7 +509,7 @@ def test_2_group_cases():
     check = np.array([[551.0735, np.nan, np.nan, np.nan, -293.9943, -845.0678, -845.0677]])
     np.testing.assert_allclose(data, check, tol)
 
-    check = np.array([[GOOD, DNU | SAT, DNU | SAT, DNU, GOOD, GOOD, GOOD]])
+    check = np.array([[GOOD, DNU | SAT, DNU | SAT, DNU | SAT, GOOD, GOOD, SAT]])
     # np.testing.assert_allclose(dq, check, tol)  # XXX double
 
     check = np.array([[38.945766, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]])
@@ -597,7 +595,7 @@ def test_one_group_ramp_suppressed_one_integration():
     check = np.array([[np.nan, np.nan, 1.0000001]])
     np.testing.assert_allclose(sdata, check, tol)
 
-    check = np.array([[DNU | SAT, DNU, GOOD]])
+    check = np.array([[DNU | SAT, DNU | SAT, GOOD]])
     # np.testing.assert_allclose(sdq, check, tol)  # XXX double
 
     check = np.array([[0.0, 0.0, 0.25]])
@@ -615,7 +613,7 @@ def test_one_group_ramp_suppressed_one_integration():
     check = np.array([[[np.nan, np.nan, 1.0000001]]])
     np.testing.assert_allclose(cdata, check, tol)
 
-    check = np.array([[[DNU | SAT, DNU, GOOD]]])
+    check = np.array([[[DNU | SAT, DNU | SAT, GOOD]]])
     # np.testing.assert_allclose(cdq, check, tol)  # XXX double
 
     check = np.array([[[0.0, 0.0, 0.25]]])
@@ -642,7 +640,7 @@ def test_one_group_ramp_not_suppressed_one_integration():
     check = np.array([[np.nan, 1.0, 1.0000001]])
     np.testing.assert_allclose(sdata, check, tol)
 
-    check = np.array([[DNU | SAT, GOOD, GOOD]])
+    check = np.array([[DNU | SAT, SAT, GOOD]])
     # np.testing.assert_allclose(sdq, check, tol)  # XXX double
 
     check = np.array([[0.0, 1.0, 0.25]])
@@ -660,7 +658,7 @@ def test_one_group_ramp_not_suppressed_one_integration():
     check = np.array([[[np.nan, 1.0, 1.0000001]]])
     np.testing.assert_allclose(cdata, check, tol)
 
-    check = np.array([[[DNU | SAT, GOOD, GOOD]]])
+    check = np.array([[[DNU | SAT, SAT, GOOD]]])
     # np.testing.assert_allclose(cdq, check, tol)  # XXX double
 
     check = np.array([[[0.0, 1, 0.25]]])
@@ -688,7 +686,7 @@ def test_one_group_ramp_suppressed_two_integrations():
     check = np.array([[1.0000001, 1.0000001, 1.0000001]])
     np.testing.assert_allclose(sdata, check, tol)
 
-    check = np.array([[GOOD, GOOD, GOOD]])
+    check = np.array([[SAT, SAT, GOOD]])
     np.testing.assert_allclose(sdq, check, tol)
 
     check = np.array([[0.125, 0.125, 0.125]])
@@ -706,7 +704,7 @@ def test_one_group_ramp_suppressed_two_integrations():
     check = np.array([[[np.nan, np.nan, 1.0000001]], [[1.0000001, 1.0000001, 1.0000001]]])
     np.testing.assert_allclose(cdata, check, tol)
 
-    check = np.array([[[DNU | SAT, DNU, GOOD]], [[GOOD, GOOD, GOOD]]])
+    check = np.array([[[DNU | SAT, DNU | SAT, GOOD]], [[GOOD, GOOD, GOOD]]])
     # np.testing.assert_allclose(cdq, check, tol)  # XXX double
 
     check = np.array([[[0.0, 0.0, 0.25]], [[0.125, 0.125, 0.25]]])
@@ -734,7 +732,7 @@ def test_one_group_ramp_not_suppressed_two_integrations():
     check = np.array([[1.0000001, 1.0000001, 1.0000001]])
     np.testing.assert_allclose(sdata, check, tol)
 
-    check = np.array([[GOOD, GOOD, GOOD]])
+    check = np.array([[SAT, SAT, GOOD]])
     np.testing.assert_allclose(sdq, check, tol)
 
     check = np.array([[0.125, 0.2, 0.125]])
@@ -752,7 +750,7 @@ def test_one_group_ramp_not_suppressed_two_integrations():
     check = np.array([[[np.nan, 1.0, 1.0000001]], [[1.0000001, 1.0000001, 1.0000001]]])
     np.testing.assert_allclose(cdata, check, tol)
 
-    check = np.array([[[DNU | SAT, GOOD, GOOD]], [[GOOD, GOOD, GOOD]]])
+    check = np.array([[[DNU | SAT, SAT, GOOD]], [[GOOD, GOOD, GOOD]]])
     # np.testing.assert_allclose(cdq, check, tol)  # XXX double
 
     check = np.array([[[0.0, 1.0, 0.25]], [[0.125, 0.25, 0.25]]])
@@ -866,7 +864,7 @@ def test_zeroframe():
     check = np.array([[48.965397, 18.628912, 47.863224]])
     np.testing.assert_allclose(sdata, check, tol, tol)
 
-    check = np.array([[GOOD, GOOD, GOOD]])
+    check = np.array([[SAT, SAT, SAT]])
     np.testing.assert_allclose(sdq, check, tol, tol)
 
     check = np.array([[0.13110262, 0.00867591, 0.29745975]])
@@ -887,7 +885,7 @@ def test_zeroframe():
     check = np.array([[[298.0626, np.nan, 652.01196]], [[18.62891, 18.62891, 18.62891]]])
     np.testing.assert_allclose(cdata, check, tol, tol)
 
-    check = np.array([[[GOOD, DNU | SAT, GOOD]], [[GOOD, GOOD, GOOD]]])
+    check = np.array([[[SAT, DNU | SAT, SAT]], [[GOOD, GOOD, GOOD]]])
     # np.testing.assert_allclose(cdq, check, tol, tol)  # XXX double
 
     check = np.array([[[1.1799237, 0.0, 6.246655]], [[0.14749046, 0.00867591, 0.31233275]]])
@@ -1463,7 +1461,7 @@ def test_one_group():
     sdata, sdq, svp, svr, serr = slopes
 
     # XXX JP-3121: this is the value from python, which may not be correct
-    chk_data = 1.9618962  
+    chk_data = 1.9618962
     chk_dq = 0
     chk_var_p = 0.02923839
     chk_var_r = 0.03470363