Merge branch 'main' of github.com:legacysurvey/legacypipe

.circleci/circleci-build-ubuntu18.04-py36/Dockerfile

@@ -88,10 +88,9 @@ ENV PYTHON=python3
 # The Tractor
 RUN git clone http://github.com/dstndstn/tractor.git tractor \
     && cd tractor \
-    && python3.6 setup-cython.py --with-ceres install --home /usr/local \
-    && make version && cp tractor/version.py /usr/local/lib/python/tractor/ \
-    && rm -R $(find . -name "*.o" -o -name "*.so") \
-    && (cd && PYTHONPATH=/usr/local/lib/python python3 -c "import tractor; print(tractor.__version__)")
+    && python3 setup.py build_ext --with-ceres --with-cython \
+    && python3 setup.py install --with-ceres --with-cython \
+    && rm -R $(find . -name "*.o" -o -name "*.so")
 
 # python = python3
 RUN ln -s /usr/bin/python3 /usr/bin/python
@@ -110,6 +109,4 @@ RUN git clone https://github.com/legacysurvey/unwise_psf.git \
 
 ENV PYTHONPATH=/usr/local/lib/python:/src/unwise_psf/py:/src/legacypipe/py
 
-# ubuntu renames the (awfully named) executable
-RUN ln -s /usr/bin/sextractor /usr/bin/sex
-
+RUN ln -s /usr/bin/sextractor /usr/bin/source-extractor

.circleci/circleci-build-ubuntu18.04-py378/Dockerfile

@@ -45,9 +45,6 @@ RUN apt -y update && apt install -y apt-utils && \
     psfex \
     && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-# ubuntu renames the (awfully named) executable
-RUN ln -s /usr/bin/sextractor /usr/bin/sex
-
 RUN python3.7 -m pip install setuptools \
  && python3.7 -m pip install pip --upgrade \
  && python3.8 -m pip install setuptools \
@@ -96,24 +93,32 @@ RUN ln -s /usr/bin/python3.7 /usr/bin/python
 
 ENV PYTHONPATH=/usr/local/lib/python:/src/unwise_psf/py:/src/legacypipe/py
 
+RUN python3.7 -m pip install wheel wheelfile \
+&&  python3.8 -m pip install wheel wheelfile
+
+# https://github.com/pypa/setuptools/issues/4501
+RUN python3.8 -m pip install setuptools==70.3.0
+
 # The Tractor
 RUN git clone http://github.com/dstndstn/tractor.git tractor \
     && cd tractor \
-    && python3.7 setup-cython.py --with-ceres install --home /usr/local --old-and-unmanageable \
-    && python3.8 setup-cython.py --with-ceres install --home /usr/local --old-and-unmanageable \
-    && make version && cp tractor/version.py /usr/local/lib/python/tractor/ \
+    && cp setup-with-ceres-and-cython.py setup.py \
+    && python3.7 -m pip install . \
+    && python3.8 -m pip install . \
     && rm -R $(find . -name "*.o" -o -name "*.so")
 
 # DUST maps
 RUN mkdir -p dust/maps \
   && cd dust/maps \
-  && wget -c http://portal.nersc.gov/project/cosmo/temp/dstn/travis-ci/maps/SFD_dust_4096_ngp.fits \
-  && wget -c http://portal.nersc.gov/project/cosmo/temp/dstn/travis-ci/maps/SFD_dust_4096_sgp.fits
+  && wget -nv http://portal.nersc.gov/project/cosmo/temp/dstn/travis-ci/maps/SFD_dust_4096_ngp.fits \
+  && wget -nv http://portal.nersc.gov/project/cosmo/temp/dstn/travis-ci/maps/SFD_dust_4096_sgp.fits
 ENV DUST_DIR=/src/dust
 
 # Unwise PSF
 RUN git clone https://github.com/legacysurvey/unwise_psf.git \
     && cd unwise_psf \
     && git checkout dr9.3
 
+RUN ln -s /usr/bin/sextractor /usr/bin/source-extractor
 
+RUN python3.8 -m pip install fitsio

docker/Dockerfile

@@ -1,21 +1,26 @@
-FROM ubuntu:22.04 as working
+FROM ubuntu:24.04 as working
 
 ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt -y update && apt install -y apt-utils \
  && apt install -y --no-install-recommends \
     gcc \
     g++ \
+    gfortran \
     make \
     patch \
     git \
     file \
     pkg-config \
+    autoconf \
+    automake \
+    libtool \
     less \
     procps \
     htop \
     gdb \
     strace \
+    valgrind \
     vim \
     less \
     nano \
@@ -26,10 +31,13 @@ RUN apt -y update && apt install -y apt-utils \
     wcslib-dev \
     wcslib-tools \
     zlib1g-dev \
+    libatlas-base-dev \
     libbz2-dev \
     libcairo2-dev \
     libcfitsio-dev \
     libcfitsio-bin \
+    libcurl4-openssl-dev \
+    libfftw3-dev \
     libgsl-dev \
     libjpeg-dev \
     libnetpbm10-dev \
@@ -39,8 +47,8 @@ RUN apt -y update && apt install -y apt-utils \
     libceres-dev \
     postgresql-common \
     libpq-dev \
-    python3.10 \
-    python3.10-dev \
+    python3.12 \
+    python3.12-dev \
     python3-pip \
     plplot-driver-cairo \
     source-extractor \
@@ -49,9 +57,6 @@ RUN apt -y update && apt install -y apt-utils \
     # Remove APT files
     && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-RUN apt -y update \
-  && apt install -y --no-install-recommends \
-      gfortran
 ENV FFLAGS=-fallow-argument-mismatch
 ENV FCFLAGS=-fallow-argument-mismatch
 
@@ -65,6 +70,8 @@ RUN wget -nv https://www.mpich.org/static/downloads/4.0.2/mpich-4.0.2.tar.gz \
   && cd .. && rm -Rf mpich-4.0.2
 # defaults to installing in /usr/local
 
+#FROM working
+
 # Pip installs
 RUN for x in \
     setuptools \
@@ -76,14 +83,15 @@ RUN for x in \
     psycopg2 \
     fitsio \
     matplotlib \
+    lacosmic \
     astropy \
     photutils \
     zmq \
     mpi4py \
-    ; do pip3 install $x; done \
+    ; do pip3 install --break-system-packages $x; done \
     && rm -Rf /root/.cache/pip
 
-RUN ln -s /usr/bin/python3.10-config /usr/local/bin/python3-config
+RUN ln -s /usr/bin/python3.12-config /usr/local/bin/python3-config
 RUN ln -s /usr/bin/python3 /usr/bin/python
 ENV PYTHON=python3
 ENV PYTHON_CONFIG=python3-config
@@ -92,21 +100,28 @@ ENV PYTHON_SCRIPT="/usr/bin/env python3"
 RUN mkdir -p /src
 WORKDIR /src
 
+# to help astrometry.net find netpbm (yuck)
 RUN ln -s /usr/include /usr/local/include/netpbm
 
 # Astrometry.net
+ENV ARCH_FLAGS=-march=x86-64-v3
 RUN git clone https://github.com/dstndstn/astrometry.net.git astrometry \
     && cd astrometry \
-    && git checkout 0.94 \
+    && git checkout 0.97 \
+    && make \
+    && make py \
     && make install INSTALL_DIR=/usr/local
 
 #FROM working
 
 # The Tractor
 RUN git clone https://github.com/dstndstn/tractor.git tractor \
     && cd tractor \
-    && pip install --install-option="--with-ceres" --install-option="--with-cython" -v .
-#    && git checkout dr10.4 \
+    && git checkout dr10.5 \
+    && python setup.py build_ext --with-ceres --with-cython --inplace \
+    && python setup.py install
+# pip no longer has "--install-option"??
+#    echo pip install --install-option="--with-ceres" --install-option="--with-cython" -v .
 
 # unwise_psf
 RUN git clone https://github.com/legacysurvey/unwise_psf.git \
@@ -135,8 +150,6 @@ ENV HOME /homedir
 RUN git config --global --add safe.directory /src/legacypipe \
  && git config --global --add safe.directory /src/unwise_psf
 
-RUN pip3 install --no-cache lacosmic
-
 RUN echo "export PS1='[container] \\u@\\h:\\w$ '" >> $HOME/.bashrc \
   # Create config files in $HOME
   && mkdir $HOME/.qdo && echo "[qdo]" > $HOME/.qdo/qdorc \
@@ -152,55 +165,40 @@ RUN echo "export PS1='[container] \\u@\\h:\\w$ '" >> $HOME/.bashrc \
 
 RUN python -O -m compileall \
     -x lib2to3 \
-    /usr/local/lib/python3.10/dist-packages \
-    /usr/lib/python3.10 \
+    /usr/local/lib/python3.12/dist-packages \
+    /usr/lib/python3.12 \
     /usr/local/lib/python \
     /src/unwise_psf/py \
     /src/legacypipe/py/legacypipe \
     /src/legacypipe/py/legacyzpts && \
     python    -m compileall \
     -x lib2to3 \
-    /usr/local/lib/python3.10/dist-packages \
-    /usr/lib/python3.10 \
+    /usr/local/lib/python3.12/dist-packages \
+    /usr/lib/python3.12 \
     /usr/local/lib/python \
     /src/unwise_psf/py \
     /src/legacypipe/py/legacypipe \
     /src/legacypipe/py/legacyzpts
 
-# ugh gross
-RUN ln -s /usr/bin/source-extractor /usr/bin/sex
-
-# update legacypipe
-RUN cd /src/legacypipe && git pull && \
-  git checkout main && git pull && git describe && \
-  python    -m compileall /src/legacypipe/py/{legacypipe,legacyzpts} && \
-  python -O -m compileall /src/legacypipe/py/{legacypipe,legacyzpts}
-
-RUN apt remove -y scamp && \
-    apt install -y autoconf libtool libfftw3-dev libatlas-base-dev libcurl4-openssl-dev && \
-    apt clean
+# Compile a local version of Scamp to pick up adding EXTNAME to the results:
+# https://github.com/astromatic/scamp/commit/d936c73d7dd08382b70cb32ceb1a535ffca81732
+RUN apt remove -y scamp
+#&& \
+#    apt install -y autoconf libtool libfftw3-dev libatlas-base-dev libcurl4-openssl-dev && \
+#    apt clean
+#
+RUN apt -y update && apt install -y automake
 
-RUN git clone https://github.com/dstndstn/scamp.git && \
+RUN git clone https://github.com/astromatic/scamp.git && \
     cd scamp && \
-    git checkout add-extname && \
     ./autogen.sh && \
     ./configure && \
     make && \
     make install && \
     make clean
-
+# 
 # update legacypipe
 RUN cd /src/legacypipe && git pull && \
-  git checkout DR10.3.2 && git describe && \
+  git checkout DR10.3.3 && git describe && \
   python    -m compileall /src/legacypipe/py/{legacypipe,legacyzpts} && \
   python -O -m compileall /src/legacypipe/py/{legacypipe,legacyzpts}
-
-RUN apt install -y valgrind
-
-ENV ARCH_FLAGS=-march=x86-64-v3
-
-#git checkout 0.96 && \
-# Astrometry.net - rebuild with generic optimization (deploying on AMD...)
-RUN cd astrometry && git checkout main && git pull && \
-    make clean && make && make py && make install INSTALL_DIR=/usr/local
-