Skip to content

Commit b42c50d

Browse files
kmacdonald-stscikmacdonald-stsci
authored
JP-3967 update for pllim (#1)
* Undoing test changes in prep to change the C-extension, as well as analyze the test failures in the LIKELY CI tests. * Commenting out a part of a test that fails. The OLS SAT flagging needs to be updated before this test works. * Updated C-extension and tests. * Updating tests for ramp fitting cases. * Removing preprocessing switches and adding a comment, cleaning up the code. * Removing unnecessary comment. * Comments on how to update slope fitter for partial saturation flagging. * Updating the parital flagging for the rate product. Updating ramp fit tests for updating flagging. * Updating partial saturation flagging and tests.
1 parent d264348 commit b42c50d

File tree

3 files changed

+44
-75
lines changed

3 files changed

+44
-75
lines changed

‎src/stcal/ramp_fitting/src/slope_fitter.c‎

Lines changed: 18 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -587,9 +587,6 @@ py_ramp_data_get_int(PyObject * rd, const char * attr);
587587
static int
588588
ramp_fit_pixel(struct ramp_data * rd, struct pixel_ramp * pr);
589589

590-
static int
591-
ramp_fit_pixel_partial_sat(struct ramp_data * rd, struct pixel_ramp * pr);
592-
593590
static int
594591
ramp_fit_pixel_rnoise_chargeloss(struct ramp_data * rd, struct pixel_ramp * pr);
595592

@@ -1282,7 +1279,6 @@ compute_integration_segments(
12821279
groupdq = pr->groupdq + integ * pr->ngroups;
12831280
}
12841281

1285-
// XXX kmacdo - SAT handling for needs to be revisited
12861282
/* If the whole integration is saturated, then no valid slope. */
12871283
if ( (!chargeloss) && (groupdq[0] & rd->sat)) {
12881284
pr->rateints[integ].dq |= rd->dnu;
@@ -2550,21 +2546,6 @@ ols_slope_fit_pixels(
25502546
if (ramp_fit_pixel(rd, pr)) {
25512547
return 1;
25522548
}
2553-
// XXX kmacdo - SAT handling for needs to be revisited
2554-
// This is probably a good place to do this to do the
2555-
// modifications desired for the partial SAT ramps.
2556-
// Loop over each integration and use:
2557-
// if 0 < pr->stats[integ].cnt_sat < pr->ngroups
2558-
// Then the ramp is only partially saturated and you
2559-
// can flag how you like and set the data values to
2560-
// what you like. I suggest a new function to do that.
2561-
//
2562-
// At this point, the SAT flag is set for fully saturated
2563-
// ramps only. Therefore you can set the partially saturated
2564-
// ramps here.
2565-
if (ramp_fit_pixel_partial_sat(rd, pr)) {
2566-
return 1;
2567-
}
25682549

25692550
if (rd->orig_gdq != Py_None) {
25702551
if (ramp_fit_pixel_rnoise_chargeloss(rd, pr)) {
@@ -2780,6 +2761,7 @@ ramp_fit_pixel(
27802761
int ret = 0;
27812762
npy_intp integ;
27822763
int sat_cnt = 0, dnu_cnt = 0;
2764+
int set_rate_sat_flag = 0;
27832765

27842766
/* Ramp fitting depends on the averaged median rate for each integration */
27852767
if (compute_median_rate(rd, pr)) {
@@ -2811,20 +2793,28 @@ ramp_fit_pixel(
28112793
dnu_cnt++;
28122794
pr->rateints[integ].slope = NAN;
28132795
}
2796+
2797+
if (rd->save_opt) {
2798+
get_pixel_ramp_integration_segments_and_pedestal(integ, pr, rd);
2799+
}
2800+
28142801
if (pr->rateints[integ].dq & rd->sat) {
28152802
sat_cnt++;
28162803
pr->rateints[integ].slope = NAN;
28172804
}
28182805

2819-
if (rd->save_opt) {
2820-
get_pixel_ramp_integration_segments_and_pedestal(integ, pr, rd);
2806+
// The partial saturation must go here to not mess up pedestal computations
2807+
if (pr->stats[integ].cnt_sat > 0) {
2808+
pr->rateints[integ].dq |= rd->sat;
2809+
set_rate_sat_flag = 1;
28212810
}
28222811
}
28232812

28242813
if (rd->nints == dnu_cnt) {
28252814
pr->rate.dq |= rd->dnu;
28262815
}
2827-
if (rd->nints == sat_cnt) {
2816+
2817+
if (sat_cnt == rd->nints) {
28282818
pr->rate.dq |= rd->sat;
28292819
}
28302820

@@ -2850,6 +2840,11 @@ ramp_fit_pixel(
28502840
pr->rate.var_err = 0.;
28512841
}
28522842

2843+
// Partial saturation flagging must be done here
2844+
if (set_rate_sat_flag) {
2845+
pr->rate.dq |= rd->sat;
2846+
}
2847+
28532848
if (!isnan(pr->rate.slope)) {
28542849
pr->rate.slope = pr->rate.slope / pr->invvar_e_sum;
28552850
}
@@ -2860,34 +2855,6 @@ ramp_fit_pixel(
28602855
return ret;
28612856
}
28622857

2863-
static int
2864-
ramp_fit_pixel_partial_sat(
2865-
struct ramp_data * rd, /* The ramp data */
2866-
struct pixel_ramp * pr) /* The pixel ramp data */
2867-
{
2868-
npy_intp integ;
2869-
int partial_sat_found = 0;
2870-
2871-
for (integ = 0; integ < pr->nints; ++integ) {
2872-
if ((pr->stats[integ].cnt_sat > 0) &&
2873-
(pr->stats[integ].cnt_sat < pr->ngroups)) {
2874-
/* Partially saturated ramp found */
2875-
partial_sat_found = 1;
2876-
pr->rateints[integ].dq |= rd->sat;
2877-
}
2878-
}
2879-
2880-
// XXX Not sure if this is the desired behavior
2881-
#if 0
2882-
if (partial_sat_found) {
2883-
pr->rate.dq |= rd->sat;
2884-
}
2885-
#endif
2886-
2887-
return 0;
2888-
}
2889-
2890-
28912858
/*
28922859
* Recompute read noise variance for ramps with the CHARGELOSS flag.
28932860
*/
@@ -3034,6 +3001,7 @@ ramp_fit_pixel_integration(
30343001
goto END;
30353002
}
30363003

3004+
// Whole ramp not usable
30373005
if (rd->ngroups == pr->stats[integ].cnt_dnu_sat) {
30383006
pr->rateints[integ].dq |= rd->dnu;
30393007
if (rd->ngroups == pr->stats[integ].cnt_sat) {

‎tests/test_ramp_fitting.py‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -507,7 +507,7 @@ def test_2_group_cases():
507507
np.testing.assert_allclose(data, check, tol)
508508

509509
check = np.array([[GOOD, DNU | SAT, DNU | SAT, DNU | SAT, GOOD, GOOD, SAT]])
510-
np.testing.assert_allclose(dq, check, tol)
510+
np.testing.assert_equal(dq, check)
511511

512512
check = np.array([[38.945766, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]])
513513
np.testing.assert_allclose(vp, check, tol)
@@ -592,7 +592,7 @@ def test_one_group_ramp_suppressed_one_integration():
592592
np.testing.assert_allclose(sdata, check, tol)
593593

594594
check = np.array([[DNU | SAT, DNU | SAT, GOOD]])
595-
np.testing.assert_allclose(sdq, check, tol)
595+
np.testing.assert_equal(sdq, check)
596596

597597
check = np.array([[0.0, 0.0, 0.25]])
598598
np.testing.assert_allclose(svp, check, tol)
@@ -610,7 +610,7 @@ def test_one_group_ramp_suppressed_one_integration():
610610
np.testing.assert_allclose(cdata, check, tol)
611611

612612
check = np.array([[[DNU | SAT, DNU | SAT, GOOD]]])
613-
np.testing.assert_allclose(cdq, check, tol)
613+
np.testing.assert_equal(cdq, check)
614614

615615
check = np.array([[[0.0, 0.0, 0.25]]])
616616
np.testing.assert_allclose(cvp, check, tol)
@@ -637,7 +637,7 @@ def test_one_group_ramp_not_suppressed_one_integration():
637637
np.testing.assert_allclose(sdata, check, tol)
638638

639639
check = np.array([[DNU | SAT, SAT, GOOD]])
640-
np.testing.assert_allclose(sdq, check, tol)
640+
np.testing.assert_equal(sdq, check)
641641

642642
check = np.array([[0.0, 1.0, 0.25]])
643643
np.testing.assert_allclose(svp, check, tol)
@@ -655,7 +655,7 @@ def test_one_group_ramp_not_suppressed_one_integration():
655655
np.testing.assert_allclose(cdata, check, tol)
656656

657657
check = np.array([[[DNU | SAT, SAT, GOOD]]])
658-
np.testing.assert_allclose(cdq, check, tol)
658+
np.testing.assert_equal(cdq, check)
659659

660660
check = np.array([[[0.0, 1, 0.25]]])
661661
np.testing.assert_allclose(cvp, check, tol)
@@ -683,7 +683,7 @@ def test_one_group_ramp_suppressed_two_integrations():
683683
np.testing.assert_allclose(sdata, check, tol)
684684

685685
check = np.array([[SAT, SAT, GOOD]])
686-
np.testing.assert_allclose(sdq, check, tol)
686+
np.testing.assert_equal(sdq, check)
687687

688688
check = np.array([[0.125, 0.125, 0.125]])
689689
np.testing.assert_allclose(svp, check, tol)
@@ -701,7 +701,7 @@ def test_one_group_ramp_suppressed_two_integrations():
701701
np.testing.assert_allclose(cdata, check, tol)
702702

703703
check = np.array([[[DNU | SAT, DNU | SAT, GOOD]], [[GOOD, GOOD, GOOD]]])
704-
np.testing.assert_allclose(cdq, check, tol)
704+
np.testing.assert_equal(cdq, check)
705705

706706
check = np.array([[[0.0, 0.0, 0.25]], [[0.125, 0.125, 0.25]]])
707707
np.testing.assert_allclose(cvp, check, tol)
@@ -729,7 +729,7 @@ def test_one_group_ramp_not_suppressed_two_integrations():
729729
np.testing.assert_allclose(sdata, check, tol)
730730

731731
check = np.array([[SAT, SAT, GOOD]])
732-
np.testing.assert_allclose(sdq, check, tol)
732+
np.testing.assert_equal(sdq, check)
733733

734734
check = np.array([[0.125, 0.2, 0.125]])
735735
np.testing.assert_allclose(svp, check, tol)
@@ -747,7 +747,7 @@ def test_one_group_ramp_not_suppressed_two_integrations():
747747
np.testing.assert_allclose(cdata, check, tol)
748748

749749
check = np.array([[[DNU | SAT, SAT, GOOD]], [[GOOD, GOOD, GOOD]]])
750-
np.testing.assert_allclose(cdq, check, tol)
750+
np.testing.assert_equal(cdq, check)
751751

752752
check = np.array([[[0.0, 1.0, 0.25]], [[0.125, 0.25, 0.25]]])
753753
np.testing.assert_allclose(cvp, check, tol)
@@ -861,7 +861,7 @@ def test_zeroframe():
861861
np.testing.assert_allclose(sdata, check, tol, tol)
862862

863863
check = np.array([[SAT, SAT, SAT]])
864-
np.testing.assert_allclose(sdq, check, tol, tol)
864+
np.testing.assert_equal(sdq, check)
865865

866866
check = np.array([[0.13110262, 0.00867591, 0.29745975]])
867867
np.testing.assert_allclose(svp, check, tol, tol)
@@ -882,7 +882,7 @@ def test_zeroframe():
882882
np.testing.assert_allclose(cdata, check, tol, tol)
883883

884884
check = np.array([[[SAT, DNU | SAT, SAT]], [[GOOD, GOOD, GOOD]]])
885-
np.testing.assert_allclose(cdq, check, tol, tol)
885+
np.testing.assert_equal(cdq, check)
886886

887887
check = np.array([[[1.1799237, 0.0, 6.246655]], [[0.14749046, 0.00867591, 0.31233275]]])
888888
np.testing.assert_allclose(cvp, check, tol, tol)
@@ -985,8 +985,8 @@ def test_only_good_0th_group():
985985
check = np.array([[37.257824, 37.257824, 496.77103]])
986986
np.testing.assert_allclose(sdata, check, tol, tol)
987987

988-
check = np.array([[GOOD, GOOD, GOOD]])
989-
np.testing.assert_allclose(sdq, check, tol, tol)
988+
check = np.array([[GOOD, SAT, SAT]])
989+
np.testing.assert_equal(sdq, check)
990990

991991
check = np.array([[0.03470363, 0.13881457, 6.169534]])
992992
np.testing.assert_allclose(svp, check, tol, tol)
@@ -1305,8 +1305,8 @@ def test_new_saturation():
13051305
check = np.array([[2.795187, 2.795632, np.nan]])
13061306
np.testing.assert_allclose(sdata, check, tol, tol)
13071307

1308-
check = np.array([[JUMP, JUMP, DNU | SAT]])
1309-
np.testing.assert_allclose(sdq, check, tol, tol)
1308+
check = np.array([[JUMP | SAT, JUMP | SAT, DNU | SAT]])
1309+
np.testing.assert_equal(sdq, check)
13101310

13111311
check = np.array([[0.00033543, 0.00043342, 0.0]])
13121312
np.testing.assert_allclose(svp, check, tol, tol)
@@ -1323,7 +1323,7 @@ def test_new_saturation():
13231323
check = np.array([[[2.7949152, 2.7956316, np.nan]], [[2.7956493, np.nan, np.nan]]])
13241324
np.testing.assert_allclose(cdata, check, tol, tol)
13251325

1326-
check = np.array([[[GOOD, JUMP, DNU | SAT]], [[JUMP, DNU | SAT, DNU | SAT]]])
1326+
check = np.array([[[GOOD, JUMP | SAT, DNU | SAT]], [[JUMP | SAT, DNU | SAT, DNU | SAT]]])
13271327
np.testing.assert_allclose(cdq, check, tol, tol)
13281328

13291329
check = np.array([[[0.00054729, 0.00043342, 0.0]], [[0.00086654, 0.0, 0.0]]])
@@ -1397,8 +1397,8 @@ def test_invalid_integrations():
13971397
check = np.array([[5434.022]])
13981398
np.testing.assert_allclose(sdata, check, tol, tol)
13991399

1400-
check = np.array([[JUMP]])
1401-
np.testing.assert_allclose(sdq, check, tol, tol)
1400+
check = np.array([[JUMP | SAT]])
1401+
np.testing.assert_equal(sdq, check)
14021402

14031403
check = np.array([[44.503918]])
14041404
np.testing.assert_allclose(svp, check, tol, tol)
@@ -1418,7 +1418,8 @@ def test_invalid_integrations():
14181418
check = np.array(
14191419
[JUMP, JUMP | DNU, JUMP | DNU, GOOD, JUMP | DNU, JUMP | DNU, JUMP | DNU, JUMP | DNU], dtype=np.uint8
14201420
)
1421-
np.testing.assert_allclose(cdq[:, 0, 0], check, tol, tol)
1421+
check |= SAT
1422+
np.testing.assert_equal(cdq[:, 0, 0], check)
14221423

14231424
check = np.array([89.007835, 0.0, 0.0, 89.007835, 0.0, 0.0, 0.0, 0.0], dtype=np.float32)
14241425
np.testing.assert_allclose(cvp[:, 0, 0], check, tol, tol)

‎tests/test_ramp_fitting_cases.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ def test_pix_1():
107107
)
108108

109109
# Set truth values for PRIMARY results:
110-
p_true = [1.8999999, JUMP, 1.05057204, 0.03454545, 1.0691562]
110+
p_true = [1.8999999, JUMP | SAT, 1.05057204, 0.03454545, 1.0691562]
111111

112112
# Set truth values for OPTIONAL results:
113113
o_true = [1.9, 56.870003, 0.03454545, 1.0691562, -3.0, 56.870003, 13.1, 0.82091206]
@@ -140,7 +140,7 @@ def test_pix_2():
140140
)
141141

142142
# Set truth values for PRIMARY results:
143-
p_true = [0.84833729, JUMP, 0.42747884, 0.00454545, 0.1781927]
143+
p_true = [0.84833729, JUMP | SAT, 0.42747884, 0.00454545, 0.1781927]
144144

145145
# Set truth values for OPTIONAL results for all segments
146146
o_true = [
@@ -225,7 +225,7 @@ def test_pix_4():
225225
)
226226

227227
# Set truth values for PRIMARY results:
228-
p_true = [1.5, GOOD, 1.047105, 0.02727273, 1.0691562]
228+
p_true = [1.5, SAT, 1.047105, 0.02727273, 1.0691562]
229229

230230
# Set truth values for OPTIONAL results:
231231
o_true = [1.5, 0.0, 0.02727273, 1.0691562, 0.0, 0.0, 13.5, 0.8318386]
@@ -426,7 +426,7 @@ def test_pix_8():
426426
)
427427

428428
# Set truth values for PRIMARY results:
429-
p_true = [0.98561335, JUMP, 0.1848883, 0.00363636, 0.03054732]
429+
p_true = [0.98561335, JUMP | SAT, 0.1848883, 0.00363636, 0.03054732]
430430

431431
# Set truth values for OPTIONAL results:
432432
o_true = [0.98561335, 9.920554, 0.00363636, 0.03054732, 16.508228, 39.383667, 14.014386, 855.78046]
@@ -544,7 +544,7 @@ def test_pix_11():
544544
)
545545

546546
# Set truth values for PRIMARY results:
547-
p_true = [1.0, GOOD, 1.042755, 0.01818182, 1.0691562]
547+
p_true = [1.0, SAT, 1.042755, 0.01818182, 1.0691562]
548548

549549
# Set truth values for OPTIONAL results:
550550
o_true = [1.0, 56.870003, 0.01818182, 1.0691562, 15.0, 56.870003, 14.0, 0.84580624]
@@ -583,7 +583,7 @@ def test_pix_12():
583583
# slope, dq, err, var_p, var_r
584584
# slope = group1 / deltatime = 15 / 10 = 1.5
585585
# dq = 2 (saturation) because group2 is saturated, but DNU is *not* set
586-
p_true = [1.5, GOOD, 1.047105, 0.027273, 1.069156]
586+
p_true = [1.5, SAT, 1.047105, 0.027273, 1.069156]
587587

588588
# Set truth values for OPTIONAL results:
589589
# slope, sig_slope, var_p, var_r, yint, sig_yint, pedestal, weights

0 commit comments

Comments
 (0)