From 829cb54ac0f293de6fc3cc7a7fa0ee99ff441bb7 Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Tue, 1 Oct 2024 11:52:27 -0400
Subject: [PATCH 01/19] changes to include resample and extract_1d into mos bg
 pipe

---
 jwst/master_background/master_background_mos_step.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/jwst/master_background/master_background_mos_step.py b/jwst/master_background/master_background_mos_step.py
index 45c641ff5c..73af6bee9b 100644
--- a/jwst/master_background/master_background_mos_step.py
+++ b/jwst/master_background/master_background_mos_step.py
@@ -9,6 +9,8 @@
 from ..flatfield import flat_field_step
 from ..pathloss import pathloss_step
 from ..photom import photom_step
+from ..resample import resample_spec_step
+from ..extract_1d import extract_1d_step
 from ..stpipe import Pipeline
 
 __all__ = ['MasterBackgroundMosStep']
@@ -60,6 +62,8 @@ class MasterBackgroundMosStep(Pipeline):
         'pathloss': pathloss_step.PathLossStep,
         'barshadow': barshadow_step.BarShadowStep,
         'photom': photom_step.PhotomStep,
+        'resample_spec': resample_spec_step.ResampleSpecStep,
+        'extract_1d': extract_1d_step.Extract1dStep,
     }
 
     # No need to prefetch. This will have been done by the parent step.

From a5d996babc27b90cb5b8d2106c01daa33606c8d6 Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Tue, 1 Oct 2024 18:49:34 -0400
Subject: [PATCH 02/19] changes entry

---
 changes/8847 | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/8847

diff --git a/changes/8847 b/changes/8847
new file mode 100644
index 0000000000..05eee27ee2
--- /dev/null
+++ b/changes/8847
@@ -0,0 +1 @@
+include resample and extract_1d into MOS background pipeline

From c21b9c2dfb922563b0e90a44f9b5042229743c81 Mon Sep 17 00:00:00 2001
From: Maria <penaguerrero@users.noreply.github.com>
Date: Wed, 2 Oct 2024 10:17:46 -0400
Subject: [PATCH 03/19] Delete changes/8847

mistake
---
 changes/8847 | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 changes/8847

diff --git a/changes/8847 b/changes/8847
deleted file mode 100644
index 05eee27ee2..0000000000
--- a/changes/8847
+++ /dev/null
@@ -1 +0,0 @@
-include resample and extract_1d into MOS background pipeline

From a4cbdb39ccededd9cf59ad355bb865a7b2e6e76c Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Wed, 2 Oct 2024 10:20:03 -0400
Subject: [PATCH 04/19] adding changes

---
 changes/8847.refpix.rst | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/8847.refpix.rst

diff --git a/changes/8847.refpix.rst b/changes/8847.refpix.rst
new file mode 100644
index 0000000000..44b6fe0d39
--- /dev/null
+++ b/changes/8847.refpix.rst
@@ -0,0 +1 @@
+Include resample and extract_1d into MOS background pipeline

From f548375d205f5a770daccb486c50dfd77a7d0c45 Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Fri, 4 Oct 2024 19:29:02 -0400
Subject: [PATCH 05/19] adding changes for JP-3748

---
 changes/8847.refpix.rst                              | 2 +-
 jwst/master_background/master_background_mos_step.py | 2 ++
 jwst/master_background/nirspec_utils.py              | 6 ++++--
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/changes/8847.refpix.rst b/changes/8847.refpix.rst
index 44b6fe0d39..f394cd8e5d 100644
--- a/changes/8847.refpix.rst
+++ b/changes/8847.refpix.rst
@@ -1 +1 @@
-Include resample and extract_1d into MOS background pipeline
+Include resample, pixel_replace, and extract_1d into MOS background pipeline
diff --git a/jwst/master_background/master_background_mos_step.py b/jwst/master_background/master_background_mos_step.py
index 73af6bee9b..7514532a4e 100644
--- a/jwst/master_background/master_background_mos_step.py
+++ b/jwst/master_background/master_background_mos_step.py
@@ -9,6 +9,7 @@
 from ..flatfield import flat_field_step
 from ..pathloss import pathloss_step
 from ..photom import photom_step
+from ..pixel_replace import pixel_replace_step
 from ..resample import resample_spec_step
 from ..extract_1d import extract_1d_step
 from ..stpipe import Pipeline
@@ -62,6 +63,7 @@ class MasterBackgroundMosStep(Pipeline):
         'pathloss': pathloss_step.PathLossStep,
         'barshadow': barshadow_step.BarShadowStep,
         'photom': photom_step.PhotomStep,
+        'pixel_replace': pixel_replace_step.PixelReplaceStep,
         'resample_spec': resample_spec_step.ResampleSpecStep,
         'extract_1d': extract_1d_step.Extract1dStep,
     }
diff --git a/jwst/master_background/nirspec_utils.py b/jwst/master_background/nirspec_utils.py
index 153bf9f504..1c9cd33fa4 100644
--- a/jwst/master_background/nirspec_utils.py
+++ b/jwst/master_background/nirspec_utils.py
@@ -91,6 +91,7 @@ def create_background_from_multislit(input_model):
     master_bkg: `~jwst.datamodels.CombinedSpecModel`
         The 1D master background spectrum created from the inputs.
     """
+    from ..pixel_replace import pixel_replace_step
     from ..resample import resample_spec_step
     from ..extract_1d import extract_1d_step
     from ..combine_1d.combine1d import combine_1d_spectra
@@ -113,8 +114,9 @@ def create_background_from_multislit(input_model):
     bkg_model.slits.extend(slits)
 
     # Apply resample_spec and extract_1d to all background slitlets
-    log.info('Applying resampling and 1D extraction to background slits')
-    resamp = resample_spec_step.ResampleSpecStep.call(bkg_model)
+    log.info('Applying pixel_replace, resampling and 1D extraction to background slits')
+    pixrepl = pixel_replace_step.PixelReplaceStep.call(bkg_model)
+    resamp = resample_spec_step.ResampleSpecStep.call(pixrepl)
     x1d = extract_1d_step.Extract1dStep.call(resamp)
 
     # Call combine_1d to combine the 1D background spectra

From f78d05ab0d88b15fbe77cab7dfaaa4a3277a412b Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Fri, 4 Oct 2024 19:34:06 -0400
Subject: [PATCH 06/19] fixing comment and adding delete array statement

---
 jwst/master_background/nirspec_utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/jwst/master_background/nirspec_utils.py b/jwst/master_background/nirspec_utils.py
index 1c9cd33fa4..af48c81603 100644
--- a/jwst/master_background/nirspec_utils.py
+++ b/jwst/master_background/nirspec_utils.py
@@ -113,7 +113,7 @@ def create_background_from_multislit(input_model):
 
     bkg_model.slits.extend(slits)
 
-    # Apply resample_spec and extract_1d to all background slitlets
+    # Apply pixel_replace, resample_spec and extract_1d to all background slitlets
     log.info('Applying pixel_replace, resampling and 1D extraction to background slits')
     pixrepl = pixel_replace_step.PixelReplaceStep.call(bkg_model)
     resamp = resample_spec_step.ResampleSpecStep.call(pixrepl)
@@ -124,6 +124,7 @@ def create_background_from_multislit(input_model):
     master_bkg = combine_1d_spectra(x1d, exptime_key='exposure_time')
 
     del bkg_model
+    del pixrepl
     del resamp
     del x1d
 

From 6d4726b7f94ddbcd2530d0fb5b73120ce9692644 Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Fri, 18 Oct 2024 16:06:22 -0400
Subject: [PATCH 07/19] modifications to update step params for mos pipe

---
 .../master_background_mos_step.py             | 13 ++++++++++++-
 jwst/master_background/nirspec_utils.py       | 19 +++++++++++++++----
 2 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/jwst/master_background/master_background_mos_step.py b/jwst/master_background/master_background_mos_step.py
index 7514532a4e..3d841c16a2 100644
--- a/jwst/master_background/master_background_mos_step.py
+++ b/jwst/master_background/master_background_mos_step.py
@@ -180,6 +180,16 @@ def set_pars_from_parent(self):
                 del pars[par]
             getattr(self, step).update_pars(pars)
 
+    def _get_steps_params(self):
+        """Get substep parameters to pass on"""
+        pixrep_pars = getattr(self, 'pixel_replace').get_pars()
+        resamp_pars = getattr(self, 'resample_spec').get_pars()
+        extr1d_pars = getattr(self, 'extract_1d').get_pars()
+        params = {'pixel_replace': pixrep_pars,
+                  'resample_spec': resamp_pars,
+                  'extract_1d': extr1d_pars}
+        return params
+
     def _classify_slits(self, data):
         """Determine how many Slits are background and source types
 
@@ -252,7 +262,8 @@ def _calc_master_background(self, data, user_background=None):
                 master_background = user_background
             else:
                 self.log.debug('Calculating 1D master background')
-                master_background = nirspec_utils.create_background_from_multislit(pre_calibrated)
+                params = self._get_steps_params()
+                master_background = nirspec_utils.create_background_from_multislit(pre_calibrated, params)
             if master_background is None:
                 self.log.debug('No master background could be calculated. Returning None')
                 return None, None
diff --git a/jwst/master_background/nirspec_utils.py b/jwst/master_background/nirspec_utils.py
index af48c81603..b8ada14c31 100644
--- a/jwst/master_background/nirspec_utils.py
+++ b/jwst/master_background/nirspec_utils.py
@@ -76,7 +76,7 @@ def map_to_science_slits(input_model, master_bkg):
     return output_model
 
 
-def create_background_from_multislit(input_model):
+def create_background_from_multislit(input_model, params):
     """Create a 1D master background spectrum from a set of
     calibrated background MOS slitlets in the input
     MultiSlitModel.
@@ -86,6 +86,9 @@ def create_background_from_multislit(input_model):
     input_model : `~jwst.datamodels.MultiSlitModel`
         The input data model containing all slit instances.
 
+    params : dict
+        Dictionary containing a dictionary of the step parameters to set
+
     Returns
     -------
     master_bkg: `~jwst.datamodels.CombinedSpecModel`
@@ -113,11 +116,19 @@ def create_background_from_multislit(input_model):
 
     bkg_model.slits.extend(slits)
 
+    # update step parameters
+    pr = pixel_replace_step.PixelReplaceStep()
+    pr.update_pars(params['pixel_replace'])
+    rs = resample_spec_step.ResampleSpecStep()
+    rs.update_pars(params['resample_spec'])
+    e1d = extract_1d_step.Extract1dStep()
+    e1d.update_pars(params['extract_1d'])
+
     # Apply pixel_replace, resample_spec and extract_1d to all background slitlets
     log.info('Applying pixel_replace, resampling and 1D extraction to background slits')
-    pixrepl = pixel_replace_step.PixelReplaceStep.call(bkg_model)
-    resamp = resample_spec_step.ResampleSpecStep.call(pixrepl)
-    x1d = extract_1d_step.Extract1dStep.call(resamp)
+    pixrepl = pr.call(bkg_model)
+    resamp = rs.call(pixrepl)
+    x1d = e1d.call(resamp)
 
     # Call combine_1d to combine the 1D background spectra
     log.info('Combining 1D background spectra into master background')

From 4d6fe063fc92650ea51614cf55285eb115e29de9 Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Mon, 21 Oct 2024 16:57:38 -0400
Subject: [PATCH 08/19] addressing PR comments

---
 .../master_background_mos_step.py             | 41 ++++++++++++-----
 jwst/master_background/nirspec_utils.py       | 46 +------------------
 2 files changed, 32 insertions(+), 55 deletions(-)

diff --git a/jwst/master_background/master_background_mos_step.py b/jwst/master_background/master_background_mos_step.py
index 3d841c16a2..e91815bcca 100644
--- a/jwst/master_background/master_background_mos_step.py
+++ b/jwst/master_background/master_background_mos_step.py
@@ -180,15 +180,28 @@ def set_pars_from_parent(self):
                 del pars[par]
             getattr(self, step).update_pars(pars)
 
-    def _get_steps_params(self):
+    def _set_steps_params(self):
         """Get substep parameters to pass on"""
-        pixrep_pars = getattr(self, 'pixel_replace').get_pars()
-        resamp_pars = getattr(self, 'resample_spec').get_pars()
-        extr1d_pars = getattr(self, 'extract_1d').get_pars()
-        params = {'pixel_replace': pixrep_pars,
-                  'resample_spec': resamp_pars,
-                  'extract_1d': extr1d_pars}
-        return params
+        steps = ['pixel_replace', 'resample_spec', 'extract_1d']
+
+        for step in steps:
+            pars = getattr(self, step).get_pars()
+            getattr(self, step).update_pars(pars)
+
+    def _extend_bg_slits(self, pre_calibrated):
+        # Copy dedicated background slitlets to a temporary model
+        bkg_model = datamodels.MultiSlitModel()
+        bkg_model.update(pre_calibrated)
+        slits = []
+        for slit in pre_calibrated.slits:
+            if nirspec_utils.is_background_msa_slit(slit):
+                self.log.info(f'Using background slitlet {slit.source_name}')
+                slits.append(slit)
+        if len(slits) == 0:
+            self.log.warning('No background slitlets found; skipping master bkg correction')
+            return None
+        bkg_model.slits.extend(slits)
+        return bkg_model
 
     def _classify_slits(self, data):
         """Determine how many Slits are background and source types
@@ -261,9 +274,15 @@ def _calc_master_background(self, data, user_background=None):
                 self.log.debug(f'User background provided {user_background}')
                 master_background = user_background
             else:
-                self.log.debug('Calculating 1D master background')
-                params = self._get_steps_params()
-                master_background = nirspec_utils.create_background_from_multislit(pre_calibrated, params)
+                self.log.info('Creating MOS master background from background slitlets')
+                self._set_steps_params()
+                bkg_model = self._extend_bg_slits(pre_calibrated)
+                bkg_model = self.pixel_replace(bkg_model)
+                bkg_model = self.resample_spec(bkg_model)
+                bkg_model = self.extract_1d(bkg_model)
+                # Call combine_1d to combine the 1D background spectra
+                master_background = nirspec_utils.create_background_from_multislit(bkg_model)
+                del bkg_model
             if master_background is None:
                 self.log.debug('No master background could be calculated. Returning None')
                 return None, None
diff --git a/jwst/master_background/nirspec_utils.py b/jwst/master_background/nirspec_utils.py
index b8ada14c31..354b99cac5 100644
--- a/jwst/master_background/nirspec_utils.py
+++ b/jwst/master_background/nirspec_utils.py
@@ -76,7 +76,7 @@ def map_to_science_slits(input_model, master_bkg):
     return output_model
 
 
-def create_background_from_multislit(input_model, params):
+def create_background_from_multislit(input_model):
     """Create a 1D master background spectrum from a set of
     calibrated background MOS slitlets in the input
     MultiSlitModel.
@@ -86,58 +86,16 @@ def create_background_from_multislit(input_model, params):
     input_model : `~jwst.datamodels.MultiSlitModel`
         The input data model containing all slit instances.
 
-    params : dict
-        Dictionary containing a dictionary of the step parameters to set
-
     Returns
     -------
     master_bkg: `~jwst.datamodels.CombinedSpecModel`
         The 1D master background spectrum created from the inputs.
     """
-    from ..pixel_replace import pixel_replace_step
-    from ..resample import resample_spec_step
-    from ..extract_1d import extract_1d_step
     from ..combine_1d.combine1d import combine_1d_spectra
 
-    log.info('Creating MOS master background from background slitlets')
-
-    # Copy dedicated background slitlets to a temporary model
-    bkg_model = datamodels.MultiSlitModel()
-    bkg_model.update(input_model)
-    slits = []
-    for slit in input_model.slits:
-        if is_background_msa_slit(slit):
-            log.info(f'Using background slitlet {slit.source_name}')
-            slits.append(slit)
-
-    if len(slits) == 0:
-        log.warning('No background slitlets found; skipping master bkg correction')
-        return None
-
-    bkg_model.slits.extend(slits)
-
-    # update step parameters
-    pr = pixel_replace_step.PixelReplaceStep()
-    pr.update_pars(params['pixel_replace'])
-    rs = resample_spec_step.ResampleSpecStep()
-    rs.update_pars(params['resample_spec'])
-    e1d = extract_1d_step.Extract1dStep()
-    e1d.update_pars(params['extract_1d'])
-
-    # Apply pixel_replace, resample_spec and extract_1d to all background slitlets
-    log.info('Applying pixel_replace, resampling and 1D extraction to background slits')
-    pixrepl = pr.call(bkg_model)
-    resamp = rs.call(pixrepl)
-    x1d = e1d.call(resamp)
-
     # Call combine_1d to combine the 1D background spectra
     log.info('Combining 1D background spectra into master background')
-    master_bkg = combine_1d_spectra(x1d, exptime_key='exposure_time')
-
-    del bkg_model
-    del pixrepl
-    del resamp
-    del x1d
+    master_bkg = combine_1d_spectra(input_model, exptime_key='exposure_time')
 
     return master_bkg
 

From 8893b48345ba7bb2763e942a5b2e22fe2e6ddaf1 Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Mon, 21 Oct 2024 17:28:18 -0400
Subject: [PATCH 09/19] adding a catch in case of None

---
 .../master_background_mos_step.py                 | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/jwst/master_background/master_background_mos_step.py b/jwst/master_background/master_background_mos_step.py
index e91815bcca..5ab00291e1 100644
--- a/jwst/master_background/master_background_mos_step.py
+++ b/jwst/master_background/master_background_mos_step.py
@@ -277,12 +277,15 @@ def _calc_master_background(self, data, user_background=None):
                 self.log.info('Creating MOS master background from background slitlets')
                 self._set_steps_params()
                 bkg_model = self._extend_bg_slits(pre_calibrated)
-                bkg_model = self.pixel_replace(bkg_model)
-                bkg_model = self.resample_spec(bkg_model)
-                bkg_model = self.extract_1d(bkg_model)
-                # Call combine_1d to combine the 1D background spectra
-                master_background = nirspec_utils.create_background_from_multislit(bkg_model)
-                del bkg_model
+                if bkg_model is not None:
+                    bkg_model = self.pixel_replace(bkg_model)
+                    bkg_model = self.resample_spec(bkg_model)
+                    bkg_model = self.extract_1d(bkg_model)
+                    # Call combine_1d to combine the 1D background spectra
+                    master_background = nirspec_utils.create_background_from_multislit(bkg_model)
+                    del bkg_model
+                else:
+                    master_background = None
             if master_background is None:
                 self.log.debug('No master background could be calculated. Returning None')
                 return None, None

From 55452ce94cbc2fe2a0dcde630de861d01c2b23eb Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Tue, 22 Oct 2024 12:03:01 -0400
Subject: [PATCH 10/19] removing unused import

---
 jwst/master_background/nirspec_utils.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/jwst/master_background/nirspec_utils.py b/jwst/master_background/nirspec_utils.py
index 354b99cac5..c2df331fe6 100644
--- a/jwst/master_background/nirspec_utils.py
+++ b/jwst/master_background/nirspec_utils.py
@@ -1,8 +1,6 @@
 import logging
 import warnings
 
-from stdatamodels.jwst import datamodels
-
 log = logging.getLogger(__name__)
 log.setLevel(logging.DEBUG)
 

From 4cf6ed537018dc19abd64282f629c74a4e59cbdc Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Tue, 12 Nov 2024 13:49:13 -0500
Subject: [PATCH 11/19] including combine_1d into mos bkgd pipeline

---
 .../master_background_mos_step.py             | 11 +++++----
 jwst/master_background/nirspec_utils.py       | 24 -------------------
 2 files changed, 6 insertions(+), 29 deletions(-)

diff --git a/jwst/master_background/master_background_mos_step.py b/jwst/master_background/master_background_mos_step.py
index eed75e1aaf..2bd1c8ddef 100644
--- a/jwst/master_background/master_background_mos_step.py
+++ b/jwst/master_background/master_background_mos_step.py
@@ -12,6 +12,7 @@
 from ..pixel_replace import pixel_replace_step
 from ..resample import resample_spec_step
 from ..extract_1d import extract_1d_step
+from ..combine_1d.combine1d import combine_1d_spectra
 from ..stpipe import Pipeline
 
 __all__ = ['MasterBackgroundMosStep']
@@ -66,6 +67,7 @@ class MasterBackgroundMosStep(Pipeline):
         'pixel_replace': pixel_replace_step.PixelReplaceStep,
         'resample_spec': resample_spec_step.ResampleSpecStep,
         'extract_1d': extract_1d_step.Extract1dStep,
+        'combine_1d': combine_1d_spectra
     }
 
     # No need to prefetch. This will have been done by the parent step.
@@ -180,9 +182,9 @@ def set_pars_from_parent(self):
                 del pars[par]
             getattr(self, step).update_pars(pars)
 
-    def _set_steps_params(self):
-        """Get substep parameters to pass on"""
-        steps = ['pixel_replace', 'resample_spec', 'extract_1d']
+    def _set_mospipe_steps_params(self):
+        """Get substep parameters for MOS pipeline to pass on"""
+        steps = ['pixel_replace', 'resample_spec', 'extract_1d', 'combine_1d']
 
         for step in steps:
             pars = getattr(self, step).get_pars()
@@ -281,8 +283,7 @@ def _calc_master_background(self, data, user_background=None):
                     bkg_model = self.pixel_replace(bkg_model)
                     bkg_model = self.resample_spec(bkg_model)
                     bkg_model = self.extract_1d(bkg_model)
-                    # Call combine_1d to combine the 1D background spectra
-                    master_background = nirspec_utils.create_background_from_multislit(bkg_model)
+                    master_background = self.combine_1d_spectra(bkg_model, exptime_key='exposure_time')
                     del bkg_model
                 else:
                     master_background = None
diff --git a/jwst/master_background/nirspec_utils.py b/jwst/master_background/nirspec_utils.py
index c2df331fe6..9b89527dc3 100644
--- a/jwst/master_background/nirspec_utils.py
+++ b/jwst/master_background/nirspec_utils.py
@@ -74,30 +74,6 @@ def map_to_science_slits(input_model, master_bkg):
     return output_model
 
 
-def create_background_from_multislit(input_model):
-    """Create a 1D master background spectrum from a set of
-    calibrated background MOS slitlets in the input
-    MultiSlitModel.
-
-    Parameters
-    ----------
-    input_model : `~jwst.datamodels.MultiSlitModel`
-        The input data model containing all slit instances.
-
-    Returns
-    -------
-    master_bkg: `~jwst.datamodels.CombinedSpecModel`
-        The 1D master background spectrum created from the inputs.
-    """
-    from ..combine_1d.combine1d import combine_1d_spectra
-
-    # Call combine_1d to combine the 1D background spectra
-    log.info('Combining 1D background spectra into master background')
-    master_bkg = combine_1d_spectra(input_model, exptime_key='exposure_time')
-
-    return master_bkg
-
-
 def correct_nrs_ifu_bkg(input_model):
     """Apply point source vs. uniform source pathloss adjustments
     to a NIRSpec IFU 2D master background array.

From 6bf4cbad8a405cf5355638ac8e1fb9e10a55be8e Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Tue, 12 Nov 2024 15:21:12 -0500
Subject: [PATCH 12/19] fixed bug in combine_1d

---
 jwst/master_background/master_background_mos_step.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/jwst/master_background/master_background_mos_step.py b/jwst/master_background/master_background_mos_step.py
index 2bd1c8ddef..e887f385dc 100644
--- a/jwst/master_background/master_background_mos_step.py
+++ b/jwst/master_background/master_background_mos_step.py
@@ -12,7 +12,7 @@
 from ..pixel_replace import pixel_replace_step
 from ..resample import resample_spec_step
 from ..extract_1d import extract_1d_step
-from ..combine_1d.combine1d import combine_1d_spectra
+from ..combine_1d import combine_1d_step
 from ..stpipe import Pipeline
 
 __all__ = ['MasterBackgroundMosStep']
@@ -67,7 +67,7 @@ class MasterBackgroundMosStep(Pipeline):
         'pixel_replace': pixel_replace_step.PixelReplaceStep,
         'resample_spec': resample_spec_step.ResampleSpecStep,
         'extract_1d': extract_1d_step.Extract1dStep,
-        'combine_1d': combine_1d_spectra
+        'combine_1d': combine_1d_step.Combine1dStep
     }
 
     # No need to prefetch. This will have been done by the parent step.
@@ -277,13 +277,13 @@ def _calc_master_background(self, data, user_background=None):
                 master_background = user_background
             else:
                 self.log.info('Creating MOS master background from background slitlets')
-                self._set_steps_params()
+                self._set_mospipe_steps_params()
                 bkg_model = self._extend_bg_slits(pre_calibrated)
                 if bkg_model is not None:
                     bkg_model = self.pixel_replace(bkg_model)
                     bkg_model = self.resample_spec(bkg_model)
                     bkg_model = self.extract_1d(bkg_model)
-                    master_background = self.combine_1d_spectra(bkg_model, exptime_key='exposure_time')
+                    master_background = self.combine_1d(bkg_model)
                     del bkg_model
                 else:
                     master_background = None

From 977695319e5392fbc01dafbaf9f73fddc3ef59fa Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Tue, 12 Nov 2024 15:26:58 -0500
Subject: [PATCH 13/19] added combine_1d to changes

---
 changes/8847.refpix.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changes/8847.refpix.rst b/changes/8847.refpix.rst
index f394cd8e5d..3fe299a962 100644
--- a/changes/8847.refpix.rst
+++ b/changes/8847.refpix.rst
@@ -1 +1 @@
-Include resample, pixel_replace, and extract_1d into MOS background pipeline
+Include resample, pixel_replace, extract_1d, and combine_1d into MOS background pipeline

From e210b1b84cae33ab20e40b605cffb9ddde98cf4a Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Wed, 13 Nov 2024 16:14:38 -0500
Subject: [PATCH 14/19] combining set_param functions

---
 .../master_background_mos_step.py             | 23 ++++++++-----------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/jwst/master_background/master_background_mos_step.py b/jwst/master_background/master_background_mos_step.py
index e887f385dc..3b06928524 100644
--- a/jwst/master_background/master_background_mos_step.py
+++ b/jwst/master_background/master_background_mos_step.py
@@ -164,11 +164,12 @@ def process(self, data):
         return result
 
     def set_pars_from_parent(self):
-        """Set substep parameters from the parents substeps"""
+        """Set substep parameters from the parents substeps when needed"""
         if not self.parent:
             return
 
-        steps = ['barshadow', 'flat_field', 'pathloss', 'photom']
+        steps = ['barshadow', 'flat_field', 'pathloss', 'photom',
+                 'pixel_replace', 'resample_spec', 'extract_1d', 'combine_1d']
         pars_to_ignore = {
             'barshadow': ['source_type'],
             'flat_field': ['save_interpolated_flat'],
@@ -177,17 +178,12 @@ def set_pars_from_parent(self):
         }
 
         for step in steps:
-            pars = getattr(self.parent, step).get_pars()
-            for par in pars_to_ignore[step] + GLOBAL_PARS_TO_IGNORE:
-                del pars[par]
-            getattr(self, step).update_pars(pars)
-
-    def _set_mospipe_steps_params(self):
-        """Get substep parameters for MOS pipeline to pass on"""
-        steps = ['pixel_replace', 'resample_spec', 'extract_1d', 'combine_1d']
-
-        for step in steps:
-            pars = getattr(self, step).get_pars()
+            if step in pars_to_ignore:
+                pars = getattr(self.parent, step).get_pars()
+                for par in pars_to_ignore[step] + GLOBAL_PARS_TO_IGNORE:
+                    del pars[par]
+            else:
+                pars = getattr(self, step).get_pars()
             getattr(self, step).update_pars(pars)
 
     def _extend_bg_slits(self, pre_calibrated):
@@ -277,7 +273,6 @@ def _calc_master_background(self, data, user_background=None):
                 master_background = user_background
             else:
                 self.log.info('Creating MOS master background from background slitlets')
-                self._set_mospipe_steps_params()
                 bkg_model = self._extend_bg_slits(pre_calibrated)
                 if bkg_model is not None:
                     bkg_model = self.pixel_replace(bkg_model)

From e220e303c3eede21529a50a37fb40d38011c4588 Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Wed, 20 Nov 2024 15:22:08 -0500
Subject: [PATCH 15/19] removing unused import

---
 jwst/master_background/nirspec_utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/jwst/master_background/nirspec_utils.py b/jwst/master_background/nirspec_utils.py
index 3bb9896447..2b27367a34 100644
--- a/jwst/master_background/nirspec_utils.py
+++ b/jwst/master_background/nirspec_utils.py
@@ -3,7 +3,6 @@
 
 from scipy.signal import medfilt
 
-from stdatamodels.jwst import datamodels
 
 log = logging.getLogger(__name__)
 log.setLevel(logging.DEBUG)

From fd095591dd961a868e21fc6c566d583acdb8457a Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Fri, 22 Nov 2024 16:01:21 -0500
Subject: [PATCH 16/19] addressing PR comments

---
 docs/jwst/master_background/description.rst   |  3 +-
 .../master_background_mos_step.py             |  7 +-
 jwst/master_background/nirspec_utils.py       |  6 +-
 .../tests/test_master_background_mos.py       | 65 +++++++++++++------
 4 files changed, 53 insertions(+), 28 deletions(-)

diff --git a/docs/jwst/master_background/description.rst b/docs/jwst/master_background/description.rst
index 96f90e82aa..75b27b8680 100644
--- a/docs/jwst/master_background/description.rst
+++ b/docs/jwst/master_background/description.rst
@@ -375,7 +375,8 @@ as follows:
    steps up through :ref:`photom <photom_step>` to all slits, treating them all as
    extended sources (appropriate for background signal), and saving the extended
    source correction arrays for each slit in an internal copy of the data model
-#. If a user-supplied master background spectrum is **not** given, the
+#. If a user-supplied master background spectrum **not** given, the
+   :ref:`pixel_replace <pixel_replace_step>`,
    :ref:`resample_spec <resample_spec_step>` and :ref:`extract_1d <extract_1d_step>`
    steps are applied to the calibrated background slits, resulting
    in extracted 1D background spectra
diff --git a/jwst/master_background/master_background_mos_step.py b/jwst/master_background/master_background_mos_step.py
index 426e32ef0e..3aaafe7722 100644
--- a/jwst/master_background/master_background_mos_step.py
+++ b/jwst/master_background/master_background_mos_step.py
@@ -12,7 +12,6 @@
 from ..pixel_replace import pixel_replace_step
 from ..resample import resample_spec_step
 from ..extract_1d import extract_1d_step
-from ..combine_1d import combine_1d_step
 from ..stpipe import Pipeline
 
 __all__ = ['MasterBackgroundMosStep']
@@ -69,7 +68,6 @@ class MasterBackgroundMosStep(Pipeline):
         'pixel_replace': pixel_replace_step.PixelReplaceStep,
         'resample_spec': resample_spec_step.ResampleSpecStep,
         'extract_1d': extract_1d_step.Extract1dStep,
-        'combine_1d': combine_1d_step.Combine1dStep
     }
 
     # No need to prefetch. This will have been done by the parent step.
@@ -135,6 +133,7 @@ def process(self, data):
                 user_background = datamodels.open(self.user_background)
                 master_background, mb_multislit, bkg_x1d_spectra = self._calc_master_background(
                     data_model, user_background)
+                # data, user_background=None, sigma_clip=3, median_kernel=1)
             elif self.use_correction_pars:
                 self.log.info('Using pre-calculated correction parameters.')
                 master_background = self.correction_pars['masterbkg_1d']
@@ -183,7 +182,7 @@ def set_pars_from_parent(self):
             return
 
         steps = ['barshadow', 'flat_field', 'pathloss', 'photom',
-                 'pixel_replace', 'resample_spec', 'extract_1d', 'combine_1d']
+                 'pixel_replace', 'resample_spec', 'extract_1d']
         pars_to_ignore = {
             'barshadow': ['source_type'],
             'flat_field': ['save_interpolated_flat'],
@@ -302,7 +301,7 @@ def _calc_master_background(
                     bkg_model = self.pixel_replace(bkg_model)
                     bkg_model = self.resample_spec(bkg_model)
                     bkg_x1d_spectra = self.extract_1d(bkg_model)
-                    master_background = nirspec_utils.create_background_from_multislit(
+                    master_background = nirspec_utils.create_background_from_multispec(
                                       bkg_x1d_spectra, sigma_clip=sigma_clip, median_kernel=median_kernel)
                 else:
                     master_background = None
diff --git a/jwst/master_background/nirspec_utils.py b/jwst/master_background/nirspec_utils.py
index 2b27367a34..3420deca6a 100644
--- a/jwst/master_background/nirspec_utils.py
+++ b/jwst/master_background/nirspec_utils.py
@@ -77,14 +77,14 @@ def map_to_science_slits(input_model, master_bkg):
     return output_model
 
 
-def create_background_from_multislit(bkg_model, sigma_clip=3, median_kernel=1):
+def create_background_from_multispec(bkg_model, sigma_clip=3, median_kernel=1):
     """Create a 1D master background spectrum from a set of
     calibrated background MOS slitlets in the input
-    MultiSlitModel.
+    MultiSpecModel.
 
     Parameters
     ----------
-    bkg_model : `~jwst.datamodels.MultiSlitModel`
+    bkg_model : `~jwst.datamodels.MultiSpecModel`
         The input data model containing all slit instances.
     sigma_clip : None or float, optional
         Optional factor for sigma clipping outliers when combining background spectra.
diff --git a/jwst/master_background/tests/test_master_background_mos.py b/jwst/master_background/tests/test_master_background_mos.py
index b683f49616..f0fc67042d 100644
--- a/jwst/master_background/tests/test_master_background_mos.py
+++ b/jwst/master_background/tests/test_master_background_mos.py
@@ -2,7 +2,7 @@
 import pytest
 from astropy.io import fits
 from astropy.table import Table
-from stdatamodels.jwst.datamodels import ImageModel
+from stdatamodels.jwst.datamodels import ImageModel, MultiSlitModel
 
 from jwst.stpipe import query_step_status
 from jwst.assign_wcs import AssignWcsStep
@@ -10,6 +10,9 @@
 from jwst.extract_2d.tests.test_nirspec import create_nirspec_hdul
 from jwst.master_background import MasterBackgroundMosStep
 from jwst.master_background import nirspec_utils
+from jwst.pixel_replace import PixelReplaceStep
+from jwst.resample import ResampleSpecStep
+from jwst.extract_1d import Extract1dStep
 
 
 def create_msa_hdul():
@@ -74,6 +77,7 @@ def nirspec_msa_metfl(tmp_path):
     hdul.close()
     return filename
 
+
 @pytest.fixture
 def nirspec_msa_extracted2d(nirspec_msa_rate, nirspec_msa_metfl):
     model = ImageModel(nirspec_msa_rate)
@@ -82,6 +86,20 @@ def nirspec_msa_extracted2d(nirspec_msa_rate, nirspec_msa_metfl):
     return model
 
 
+def mk_multispec(model):
+    specs_model = MultiSlitModel()
+    specs_model.update(model)
+    slits = []
+    for slit in model.slits:
+        if nirspec_utils.is_background_msa_slit(slit):
+            slits.append(slit)
+    specs_model.slits.extend(slits)
+    specs_model = PixelReplaceStep.call(specs_model)
+    specs_model = ResampleSpecStep.call(specs_model)
+    specs_model = Extract1dStep.call(specs_model)
+    return specs_model
+
+
 def test_master_background_mos(nirspec_msa_extracted2d):
     model = nirspec_msa_extracted2d
 
@@ -97,43 +115,47 @@ def test_master_background_mos(nirspec_msa_extracted2d):
     # Check that a background was subtracted from the science data
     assert not np.allclose(sci_orig, sci_bkgsub)
 
-    model.close()
-    result.close()
+    del model
+    del result
 
 
-def test_create_background_from_multislit(nirspec_msa_extracted2d):
+def test_create_background_from_multispec(nirspec_msa_extracted2d):
     model = nirspec_msa_extracted2d
 
-    # Insert a outliers into one of the background spectra
+    # Insert outliers into one of the background spectra
     nypix = len(model.slits[0].data)
     nxpix = len(model.slits[0].data)
-    model.slits[0].data[nypix//2,nxpix//2-1:nxpix//2+1] = 10
+    model.slits[0].data[nypix//2, nxpix//2-1:nxpix//2+1] = 10
+
+    specs_model = mk_multispec(model)
 
     # First check that we can make a master background from the inputs
 
     # Check that with sigma_clip=None, the outlier is retained
-    master_background, _ = nirspec_utils.create_background_from_multislit(
-        model, sigma_clip=None)
+    master_background = nirspec_utils.create_background_from_multispec(
+        specs_model, sigma_clip=None)
     assert np.any(master_background.spec[0].spec_table['surf_bright'] > 1)
 
     # Confirm that using a median_filter will filter out the outlier
-    master_background, _ = nirspec_utils.create_background_from_multislit(
-        model, median_kernel=4)
+    master_background = nirspec_utils.create_background_from_multispec(
+        specs_model, median_kernel=4)
     assert np.allclose(master_background.spec[0].spec_table['surf_bright'], 1)
 
     # Confirm that using a sigma clipping when combining background spectra
     # removes the outlier
-    master_background, _ = nirspec_utils.create_background_from_multislit(
-        model, sigma_clip=3)
+    master_background = nirspec_utils.create_background_from_multispec(
+        specs_model, sigma_clip=3)
     assert np.allclose(master_background.spec[0].spec_table['surf_bright'], 1)
 
-    model.close()
+    del model
+    del specs_model
+
 
 def test_map_to_science_slits(nirspec_msa_extracted2d):
     model = nirspec_msa_extracted2d
+    specs_model = mk_multispec(model)
 
-    master_background, _ = nirspec_utils.create_background_from_multislit(
-        model)
+    master_background = nirspec_utils.create_background_from_multispec(specs_model)
 
     # Check that the master background is expanded to the shape of the input slits
     mb_multislit = nirspec_utils.map_to_science_slits(model, master_background)
@@ -145,13 +167,15 @@ def test_map_to_science_slits(nirspec_msa_extracted2d):
     nonzero = slit_data != 0
     assert np.allclose(slit_data[nonzero], 1)
 
-    model.close()
+    del model
+    del specs_model
+
 
 def test_apply_master_background(nirspec_msa_extracted2d):
     model = nirspec_msa_extracted2d
+    specs_model = mk_multispec(model)
 
-    master_background, _ = nirspec_utils.create_background_from_multislit(
-    model)
+    master_background = nirspec_utils.create_background_from_multispec(specs_model)
     mb_multislit = nirspec_utils.map_to_science_slits(model, master_background)
 
     result = nirspec_utils.apply_master_background(model, mb_multislit, inverse=False)
@@ -175,7 +199,8 @@ def test_apply_master_background(nirspec_msa_extracted2d):
     assert np.any(diff != 0)
     assert np.allclose(diff[diff != 0], -1)
 
-    model.close()
-    result.close()
+    del model
+    del result
+    del specs_model
 
 

From 9ec2783231bf2f42b52a639314cd1a5f8f4b928d Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Fri, 22 Nov 2024 16:05:57 -0500
Subject: [PATCH 17/19] fixed typo

---
 docs/jwst/master_background/description.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/jwst/master_background/description.rst b/docs/jwst/master_background/description.rst
index 75b27b8680..d54b1eb0e6 100644
--- a/docs/jwst/master_background/description.rst
+++ b/docs/jwst/master_background/description.rst
@@ -375,7 +375,7 @@ as follows:
    steps up through :ref:`photom <photom_step>` to all slits, treating them all as
    extended sources (appropriate for background signal), and saving the extended
    source correction arrays for each slit in an internal copy of the data model
-#. If a user-supplied master background spectrum **not** given, the
+#. If a user-supplied master background spectrum is **not** given, the
    :ref:`pixel_replace <pixel_replace_step>`,
    :ref:`resample_spec <resample_spec_step>` and :ref:`extract_1d <extract_1d_step>`
    steps are applied to the calibrated background slits, resulting

From 9f57dbf8d0fa71a9f5f9188dedd588cfc06bdae0 Mon Sep 17 00:00:00 2001
From: Maria <penaguerrero@users.noreply.github.com>
Date: Tue, 26 Nov 2024 18:59:53 -0500
Subject: [PATCH 18/19] Delete changes/8847.refpix.rst

removing this because it is for the wrong step
---
 changes/8847.refpix.rst | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 changes/8847.refpix.rst

diff --git a/changes/8847.refpix.rst b/changes/8847.refpix.rst
deleted file mode 100644
index 3fe299a962..0000000000
--- a/changes/8847.refpix.rst
+++ /dev/null
@@ -1 +0,0 @@
-Include resample, pixel_replace, extract_1d, and combine_1d into MOS background pipeline

From 53cdadd73e479377e43f1fc29ce27600b781ab10 Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Tue, 26 Nov 2024 19:10:38 -0500
Subject: [PATCH 19/19] addressing rebase bugs and comments

---
 changes/8847.master_background.rst            |  1 +
 .../master_background_mos_step.py             | 19 +++++++------------
 2 files changed, 8 insertions(+), 12 deletions(-)
 create mode 100644 changes/8847.master_background.rst

diff --git a/changes/8847.master_background.rst b/changes/8847.master_background.rst
new file mode 100644
index 0000000000..f1b516a1f2
--- /dev/null
+++ b/changes/8847.master_background.rst
@@ -0,0 +1 @@
+Include resample, pixel_replace and extract_1d into MOS background pipeline
diff --git a/jwst/master_background/master_background_mos_step.py b/jwst/master_background/master_background_mos_step.py
index 3aaafe7722..d5ad86b711 100644
--- a/jwst/master_background/master_background_mos_step.py
+++ b/jwst/master_background/master_background_mos_step.py
@@ -133,7 +133,6 @@ def process(self, data):
                 user_background = datamodels.open(self.user_background)
                 master_background, mb_multislit, bkg_x1d_spectra = self._calc_master_background(
                     data_model, user_background)
-                # data, user_background=None, sigma_clip=3, median_kernel=1)
             elif self.use_correction_pars:
                 self.log.info('Using pre-calculated correction parameters.')
                 master_background = self.correction_pars['masterbkg_1d']
@@ -181,8 +180,7 @@ def set_pars_from_parent(self):
         if not self.parent:
             return
 
-        steps = ['barshadow', 'flat_field', 'pathloss', 'photom',
-                 'pixel_replace', 'resample_spec', 'extract_1d']
+        steps = ['barshadow', 'flat_field', 'pathloss', 'photom']
         pars_to_ignore = {
             'barshadow': ['source_type'],
             'flat_field': ['save_interpolated_flat'],
@@ -191,12 +189,9 @@ def set_pars_from_parent(self):
         }
 
         for step in steps:
-            if step in pars_to_ignore:
-                pars = getattr(self.parent, step).get_pars()
-                for par in pars_to_ignore[step] + GLOBAL_PARS_TO_IGNORE:
-                    del pars[par]
-            else:
-                pars = getattr(self, step).get_pars()
+            pars = getattr(self.parent, step).get_pars()
+            for par in pars_to_ignore[step] + GLOBAL_PARS_TO_IGNORE:
+                del pars[par]
             getattr(self, step).update_pars(pars)
 
     def _extend_bg_slits(self, pre_calibrated):
@@ -298,9 +293,9 @@ def _calc_master_background(
                 self.log.info('Creating MOS master background from background slitlets')
                 bkg_model = self._extend_bg_slits(pre_calibrated)
                 if bkg_model is not None:
-                    bkg_model = self.pixel_replace(bkg_model)
-                    bkg_model = self.resample_spec(bkg_model)
-                    bkg_x1d_spectra = self.extract_1d(bkg_model)
+                    bkg_model = self.pixel_replace.run(bkg_model)
+                    bkg_model = self.resample_spec.run(bkg_model)
+                    bkg_x1d_spectra = self.extract_1d.run(bkg_model)
                     master_background = nirspec_utils.create_background_from_multispec(
                                       bkg_x1d_spectra, sigma_clip=sigma_clip, median_kernel=median_kernel)
                 else: