Merge pull request #1568 from d-ylee/update-wmcore-dev

Update wmcore-dev

docker/pypi/wmcore-dev/.gitignore

@@ -1,5 +1,8 @@
 contents/
 ContainerScripts/LatestUnitTests
 .DS_Store
+examples
+
+# virtualenvs
 */**/venv
-requirements.txt
+.venv

docker/pypi/wmcore-dev/Dockerfile

@@ -1,6 +1,7 @@
-# FROM --platform=linux/amd64 registry.cern.ch/cmsweb/alma-base:alma9-20240305
 FROM --platform=linux/amd64 registry.cern.ch/cmsweb/cmsweb-alma9-base:20240530
 
+LABEL version="0.2.0"
+
 # Install cern repo
 RUN dnf install -y 'dnf-command(config-manager)' && \
     dnf config-manager --add-repo https://linuxsoft.cern.ch/cern/alma/9/devel/x86_64/os
@@ -40,6 +41,8 @@ RUN dnf install -y \
         voms-clients-cpp && \
     dnf clean all
 
+# Create the working directory
+RUN mkdir -p /home/cmsbld
 WORKDIR /home/cmsbld
 
 # get requirements.txt and remove gfal2 to use RPM gfal2
@@ -52,11 +55,7 @@ RUN curl -O https://raw.githubusercontent.com/dmwm/WMCore/master/requirements.tx
 
 RUN pip3 install --no-cache-dir --upgrade pip setuptools wheel && \
     pip3 install --no-cache-dir --ignore-installed -r /home/cmsbld/requirements-dev.txt
-#    pip3 install --no-cache-dir --ignore-installed -r /home/cmsbld/WMCore/requirements-dev.txt
 
-ADD TestScripts /home/cmsbld/TestScripts
-ADD ContainerScripts /home/cmsbld/ContainerScripts
-ADD etc /home/cmsbld/etc
 ADD entrypoint.sh /entrypoint.sh
 
 ENV X509_HOST_CERT=/data/certs/servicecert.pem

docker/pypi/wmcore-dev/Makefile

@@ -0,0 +1,23 @@
+VERSION=0.2.0
+DATE := $(shell date +%Y%m%d)
+#DATE = 20241202
+
+build:
+	docker buildx build --platform=linux/amd64 --tag registry.cern.ch/cmsweb/wmcore-dev .
+
+build-no-cache:
+	docker buildx build --no-cache --platform=linux/amd64 --tag registry.cern.ch/cmsweb/wmcore-dev .
+
+push:
+	docker tag registry.cern.ch/cmsweb/wmcore-dev registry.cern.ch/cmsweb/wmcore-dev:$(DATE)
+	docker push registry.cern.ch/cmsweb/wmcore-dev:$(DATE)
+	docker tag registry.cern.ch/cmsweb/wmcore-dev registry.cern.ch/cmsweb/wmcore-dev:latest
+	docker push registry.cern.ch/cmsweb/wmcore-dev:latest
+
+push-preprod:
+	docker tag registry.cern.ch/cmsweb/wmcore-dev registry.cern.ch/cmsweb/wmcore-dev:$(VERSION)
+	docker push registry.cern.ch/cmsweb/wmcore-dev:$(VERSION)
+
+push-stable:
+	docker tag registry.cern.ch/cmsweb/wmcore-dev registry.cern.ch/cmsweb/wmcore-dev:$(VERSION)-stable
+	docker push registry.cern.ch/cmsweb/wmcore-dev:$(VERSION)-stable

docker/pypi/wmcore-dev/README.md

@@ -0,0 +1,17 @@
+# wmcore-dev: WMCore development image
+
+## Overview
+This image is used for running unit tests and linting for the [WMCore](https://github.com/dmwm/WMCore) repository. This image is currently used by the Jenkins CI/CD pipelines defined in [WMCore-Jenkins](https://github.com/dmwm/WMCore-Jenkins).
+
+## Usage
+1. To build the image, run `make build`
+2. Push to the CERN registry with `make push`, `make push-preprod`, or `make push-stable`
+  * Special account privileges are needed to push to the CERN registry.
+  * `*-stable` tagged releases are not automatically cleared from the registry.
+  * Currently, the date is used as a tag for `make push`, while `make push-preprod` and `make push-stable` uses a version number.
+    * `make push-preprod`: Pushes the current version
+    * `make push-stable`: Pushes the current version, appended with `-stable`
+
+## Maintainers
+[@d-ylee](https://github.com/d-ylee)
+[@khurtado](https://github.com/khurtado)

docker/pypi/wmcore-dev/TestScripts/test-wmcorepy3.sh

@@ -15,6 +15,9 @@ echo "Sourcing a python3 unittest environment"
 # load wmagent secrets
 . /data/admin/wmagent/WMAgent.secrets
 
+# Load CMS environment from CVMFS
+source /cvmfs/cms.cern.ch/cmsset_default.sh
+
 # This is what the production wmagent-mariadb image defines (via WMA_DATABASE) has as of 2024/07/15
 # TODO: We should either make an option in the mariadb image to define the default WMA_DATABASE or make a custom test mariadb image
 MDB_UNITTEST_DB=wmagent

docker/pypi/wmcore-dev/entrypoint.sh

@@ -2,23 +2,28 @@
 
 # add WMCore repository, if in Jenkins
 if [[ -z "${BUILD_ID}" ]]; then
-    echo "No BUILD_ID set"
+    echo "No BUILD_ID set, we in dev mode"
 else
-    echo "BUILD_ID set, cloning dmwm/WMCore"
-    git clone https://github.com/dmwm/WMCore
-
+    echo "BUILD_ID set, setting up for CI/CD"
     # give proper permissions to home directory
-    chown -R ${MY_ID}:${MY_GROUP} /home/cmsbld
+    #chown -R ${MY_ID}:${MY_GROUP} /home/cmsbld
 
-    USERN=$(id -un ${MY_ID})
+    #USERN=$(id -un ${MY_ID})
 
-    su - $USERN
-fi
+    pushd /home/cmsbld
 
-pushd /home/cmsbld
+    # clone main repo
+    git clone https://github.com/dmwm/WMCore
 
-# clone jenkins-test scripts
-git clone https://github.com/d-ylee/jenkins-test
-popd
+    # clone jenkins-test scripts
+    git clone https://github.com/dmwm/WMCore-Jenkins
+
+    # TODO: Move scripts to a more sensible location in WMCore-Jenkins
+    cp -r WMCore-Jenkins/docker/wmcore-dev/TestScripts .
+    cp -r WMCore-Jenkins/docker/wmcore-dev/ContainerScripts .
+    cp -r WMCore-Jenkins/docker/wmcore-dev/etc .
+    popd
+
+fi
 
 $@