Merge branch 'main' into flux_drop

.github/workflows/build.yml

@@ -21,7 +21,7 @@ jobs:
   build:
     needs: [ check ]
     if: always() && (needs.check.result == 'success' || needs.check.result == 'skipped')
-    uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish.yml@d68193b68216da64eafaa618f53c59f5d271c56e  # v1.14.0
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish.yml@9f1f43251dde69da8613ea8e11144f05cdea41d5  # v1.15.0
     with:
       upload_to_pypi: ${{ (github.event_name == 'release') && (github.event.action == 'released') }}
       targets: |

.github/workflows/ci.yml

@@ -20,8 +20,8 @@ on:
         type: string
         required: false
         default: ''
-      crds_server:
-        description: CRDS server
+      crds_server_url:
+        description: CRDS server URL
         type: string
         required: false
         default: https://jwst-crds.stsci.edu
@@ -40,34 +40,23 @@ jobs:
         python-version: '3.12'
     - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd  # v3.0.1
   check:
-    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d68193b68216da64eafaa618f53c59f5d271c56e  # v1.14.0
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@9f1f43251dde69da8613ea8e11144f05cdea41d5  # v1.15.0
     with:
       default_python: "3.12"
       envs: |
         - linux: check-dependencies
         - linux: check-types
-  latest_crds_contexts:
-    uses: ./.github/workflows/contexts.yml
-  crds_context:
-    needs: [ latest_crds_contexts ]
-    runs-on: ubuntu-latest
-    steps:
-      - id: context
-        run: echo context=${{ github.event_name == 'workflow_dispatch' && (inputs.crds_context != '' && inputs.crds_context || needs.latest_crds_contexts.outputs.jwst) || needs.latest_crds_contexts.outputs.jwst }} >> $GITHUB_OUTPUT
-    outputs:
-      context: ${{ steps.context.outputs.context }}
   test:
-    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d68193b68216da64eafaa618f53c59f5d271c56e  # v1.14.0
-    needs: [ crds_context ]
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@9f1f43251dde69da8613ea8e11144f05cdea41d5  # v1.15.0
     with:
       setenv: |
         CRDS_PATH: /tmp/data/crds_cache
-        CRDS_SERVER_URL: ${{ github.event_name == 'workflow_dispatch' && inputs.crds_server || 'https://jwst-crds.stsci.edu' }}
-        CRDS_CONTEXT: ${{ needs.crds_context.outputs.context }}
+        CRDS_SERVER_URL: ${{ inputs.crds_server_url || 'https://jwst-crds.stsci.edu' }}
+        CRDS_CONTEXT: ${{ inputs.crds_context || 'jwst-edit' }}
         CRDS_CLIENT_RETRY_COUNT: 3
         CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
       cache-path: /tmp/data/crds_cache
-      cache-key: crds-${{ needs.crds_context.outputs.context }}
+      cache-key: crds-${{ inputs.crds_context || 'jwst-edit' }}
       envs: |
         - linux: py310-oldestdeps-xdist-cov
           pytest-results-summary: true

.github/workflows/ci_cron.yml

@@ -11,8 +11,8 @@ on:
         type: string
         required: false
         default: ''
-      crds_server:
-        description: CRDS server
+      crds_server_url:
+        description: CRDS server URL
         type: string
         required: false
         default: https://jwst-crds.stsci.edu
@@ -22,25 +22,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  latest_crds_contexts:
-    uses: ./.github/workflows/contexts.yml
-  crds_context:
-    needs: [ latest_crds_contexts ]
-    runs-on: ubuntu-latest
-    steps:
-      - id: context
-        run: echo context=${{ github.event_name == 'workflow_dispatch' && (inputs.crds_context != '' && inputs.crds_context || needs.latest_crds_contexts.outputs.jwst) || needs.latest_crds_contexts.outputs.jwst }} >> $GITHUB_OUTPUT
-    outputs:
-      context: ${{ steps.context.outputs.context }}
   test:
     if: (github.repository == 'spacetelescope/jwst' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run scheduled tests')))
-    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d68193b68216da64eafaa618f53c59f5d271c56e  # v1.14.0
-    needs: [ crds_context ]
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@9f1f43251dde69da8613ea8e11144f05cdea41d5  # v1.15.0
     with:
       setenv: |
         CRDS_PATH: /tmp/crds_cache
-        CRDS_SERVER_URL: ${{ github.event_name == 'workflow_dispatch' && inputs.crds_server || 'https://jwst-crds.stsci.edu' }}
-        CRDS_CONTEXT: ${{ needs.crds_context.outputs.context }}
+        CRDS_SERVER_URL: ${{ inputs.crds_server_url || 'https://jwst-crds.stsci.edu' }}
+        CRDS_CONTEXT: ${{ inputs.crds_context || 'jwst-edit' }}
         CRDS_CLIENT_RETRY_COUNT: 3
         CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
       cache-path: /tmp/crds_cache

.github/workflows/tests_devdeps.yml

@@ -20,8 +20,8 @@ on:
         type: string
         required: false
         default: ''
-      crds_server:
-        description: CRDS server
+      crds_server_url:
+        description: CRDS server URL
         type: string
         required: false
         default: https://jwst-crds.stsci.edu
@@ -31,25 +31,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  latest_crds_contexts:
-    uses: ./.github/workflows/contexts.yml
-  crds_context:
-    needs: [ latest_crds_contexts ]
-    runs-on: ubuntu-latest
-    steps:
-      - id: context
-        run: echo context=${{ github.event_name == 'workflow_dispatch' && (inputs.crds_context != '' && inputs.crds_context || needs.latest_crds_contexts.outputs.jwst) || needs.latest_crds_contexts.outputs.jwst }} >> $GITHUB_OUTPUT
-    outputs:
-      context: ${{ steps.context.outputs.context }}
   test:
     if: (github.repository == 'spacetelescope/jwst' && (github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'run devdeps tests')))
-    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@d68193b68216da64eafaa618f53c59f5d271c56e  # v1.14.0
-    needs: [ crds_context ]
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@9f1f43251dde69da8613ea8e11144f05cdea41d5  # v1.15.0
     with:
       setenv: |
         CRDS_PATH: /tmp/data/crds_cache
-        CRDS_SERVER_URL: ${{ github.event_name == 'workflow_dispatch' && inputs.crds_server || 'https://jwst-crds.stsci.edu' }}
-        CRDS_CONTEXT: ${{ needs.crds_context.outputs.context }}
+        CRDS_SERVER_URL: ${{ inputs.crds_server_url || 'https://jwst-crds.stsci.edu' }}
+        CRDS_CONTEXT: ${{ inputs.crds_context || 'jwst-edit' }}
         CRDS_CLIENT_RETRY_COUNT: 3
         CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
       cache-path: /tmp/data/crds_cache

changes/8846.ami.rst

@@ -0,0 +1 @@
+Change how AMI observables are averaged: average fringe quantities before calculating additional observables. Update their error calculation: use covariance of amplitudes/phases (and derived quantities) and standard error of the mean. Code now expects an ASDF filename string for user-supplied affine2d and bandpass input arguments. Example file creation in documentation.

changes/8957.general.rst

@@ -0,0 +1 @@
+remove ``okify_regtests`` script (move to ``ci_watson``)

changes/8978.assign_wcs.rst

@@ -0,0 +1 @@
+Fix negative SHUTTRID values under numpy 2

docs/jwst/ami_analyze/description.rst

@@ -36,7 +36,7 @@ other options:
                     rotation search values. The default setting of '-3 3 1'
                     results in search values of [-3, -2, -1, 0, 1, 2, 3].
 
-:--bandpass: Synphot spectrum or suitable array to override filter/source 
+:--bandpass: ASDF file containing suitable array to override filter/source 
              (default=None)
 
 :--usebp: If True, exclude pixels marked DO_NOT_USE from fringe fitting 
@@ -47,10 +47,73 @@ other options:
 :--chooseholes: If not None, fit only certain fringes e.g. ['B4','B5','B6','C2']
                 (default=None)
 
-:--affine2d: User-defined Affine2d object (default=None)
+:--affine2d: ASDF file containing user-defined affine parameters (default=None)
 
 :--run_bpfix: Run Fourier bad pixel fix on cropped data (default=True)
-
+
+
+Creating ASDF files
+^^^^^^^^^^^^^^^^^^^
+The optional arguments `bandpass` and `affine2d` must be written to `ASDF <https://asdf-standard.readthedocs.io/>`_ 
+files to be used by the step. The step expects the contents to be stored with particular keys but the format is not currently
+enforced by a schema; incorrect ASDF file contents will cause the step to revert back to the defaults for each argument.
+
+Examples of how to create ASDF files containing the properly formatted information for each of the arguments follows.
+
+.. code-block:: python
+
+   # Create a F480M filter + Vega bandpass ASDF file
+
+   import asdf
+   from jwst.ami import utils
+   from stdatamodels.jwst import datamodels
+   from synphot import SourceSpectrum
+
+   # F480M throughput reference file from JWST CRDS
+   throughput_file = 'jwst_niriss_throughput_0012.fits'
+   nspecbin=19
+   throughput_model = datamodels.open(throughput_file)
+
+   filt_spec = utils.get_filt_spec(throughput_model)
+   src_spec = SourceSpectrum.from_vega()  
+   bandpass = utils.combine_src_filt(filt_spec,
+                                    src_spec,
+                                    trim=0.01,
+                                    nlambda=nspecbin)
+
+   # This bandpass has shape (19, 2); each row is [throughput, wavelength]
+   asdf_name = 'bandpass_f480m_vega.asdf'
+   tree = {"bandpass": bandpass}
+   with open(asdf_name, 'wb') as fh:
+        af = asdf.AsdfFile(tree)
+        af.write_to(fh)
+   af.close()
+   throughput_model.close()
+
+
+.. code-block:: python
+
+   # Create an affine transform ASDF file to use for the model
+
+   import asdf
+   tree = {
+         'mx': 1., # dimensionless x-magnification
+         'my': 1., # dimensionless y-magnification
+         'sx': 0., # dimensionless x shear
+         'sy': 0., # dimensionless y shear
+         'xo': 0., # x-offset in pupil space
+         'yo': 0., # y-offset in pupil space
+         'rotradccw': None 
+         }
+
+   affineasdf = 'affine.asdf'
+
+   with open(affineasdf, 'wb') as fh:
+        af = asdf.AsdfFile(tree)
+        af.write_to(fh)
+   af.close()
+
+
 
 Inputs
 ------
@@ -70,7 +133,7 @@ The ``ami_analyze`` step itself does not accept an ASN as input.
 Outputs
 -------
 
-The ``ami_analyze`` step produces three output files. The first two (``_ami-oi.fits`` and ``_amimulti-oi.fits``) contain the interferometric observables, and the third (``_amilg.fits``) contains the data, LG model, and residuals.  These are described in more detail below.
+The ``ami_analyze`` step produces three output files. The first two (``_ami-oi.fits`` and ``_amimulti-oi.fits``) contain the interferometric observables, and the third (``_amilg.fits``) contains the data, LG model, and residuals. All products contain a ``PRIMARY`` HDU containing header keywords but no science data, as well as an ``ASDF`` extension. The files are described in more detail below.
 
 The output file name syntax is exposure-based, using the input file name as the root, with
 the addition of the association candidate ID and the "_ami-oi", "_amimulti-oi", or "amilg" product type suffix, e.g.
@@ -81,20 +144,19 @@ Interferometric observables
 :Data model: `~jwst.datamodels.AmiOIModel`
 :File suffix: _ami-oi.fits, _amimulti-oi.fits
 
-. The inteferometric observables are saved as OIFITS files, a registered FITS format
+The inteferometric observables are saved as OIFITS files, a registered FITS format
 for optical interferometry, containing the following list of extensions:
 
 1)  ``OI_ARRAY``: AMI subaperture information  
 2)  ``OI_TARGET``: target properties  
-3)  ``OI_T3``: extracted closure amplitudes, phases 
+3)  ``OI_T3``: extracted closure amplitudes, triple-product phases 
 4)  ``OI_VIS``: extracted visibility (fringe) amplitudes, phases
 5)  ``OI_VIS2``: squared visibility (fringe) amplitudes
 6)  ``OI_WAVELENGTH``: filter information
 
 For more information on the format and contents of OIFITS files, see the `OIFITS2 standard <https://doi.org/10.1051/0004-6361/201526405>`_.
 
-The _ami-oi.fits file contains tables of median observables over all integrations of the input file.  Errors 
-are computed as the sigma-clipped standard deviation over integrations.
+The _ami-oi.fits file contains tables of observables averaged over all integrations of the input file. The error is taken to be the standard error of the mean, where the variance is the covariance between amplitudes and phases (e.g. fringe amplitudes and fringe phases, closure phases and triple-product amplitudes).
 The _amimulti-oi.fits file contains observables for each integration, and does not contain error estimates. The
 structure is the same as the _ami-oi.fits file, but the following data columns are 2D, with the second dimension being 
 the number of integrations: "PISTONS", "PIST_ERR", "VISAMP", "VISAMPERR", "VISPHI", "VISPHIERR", "VIS2DATA", "VIS2ERR", "T3AMP", "T3AMPERR", "T3PHI", "T3PHIERR".

docs/jwst/pipeline/calwebb_ami3.rst

@@ -7,8 +7,7 @@ calwebb_ami3: Stage 3 Aperture Masking Interferometry (AMI) Processing
 :Alias: calwebb_ami3
 
 The stage 3 AMI pipeline is applied to associations of calibrated NIRISS AMI exposures.
-It computes fringe parameters for individual exposures, averages the fringe results from
-multiple exposures, and, optionally, corrects science target fringe parameters using the
+It computes fringe parameters for individual exposures, and, optionally, corrects science target fringe parameters using the
 fringe results from reference PSF targets.
 The steps applied by the ``calwebb_ami3`` pipeline are shown below.
 
@@ -83,15 +82,14 @@ Interferometric observables
 :Data model: `~jwst.datamodels.AmiOIModel`
 :File suffix: _ami-oi.fits
 
-
-For every input exposure, the fringe parameters and closure phases calculated
+For every input exposure, the interferometric observables calculated
 by the :ref:`ami_analyze <ami_analyze_step>` step are saved to an "_ami-oi.fits" product file,
-which is a FITS table of median observables over all integrations of the input file.
+which is a FITS table of averaged observables over all integrations of the input file.
 Product names use the input "_calints" exposure-based file name, with the association candidate ID
 included and the product type changed to "_ami-oi.fits", e.g.
 "jw93210001001_03101_00001_nis_a0003_ami-oi.fits."
 
-Normalized Interferometric Observables
+Normalized interferometric observables
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 :Data model: `~jwst.datamodels.AmiOIModel`
 :File suffix: _aminorm-oi.fits
@@ -102,3 +100,8 @@ via the :ref:`ami_normalize <ami_normalize_step>` step, and will be saved to an
 product file. This file has the same FITS table format as the "_ami-oi.fits" products.
 The file name root uses the source-based output product name given in the ASN file,
 e.g. "jw93210-a0003_t001_niriss_f480m-nrm_aminorm-oi.fits."
+
+.. note:: 
+
+   Users may wish to run the :ref:`ami_analyze step <ami_analyze_step>` separately for access to flexible input arguments and to save additional diagnostic output products. See the step documentation for more details.
+