Skip to content

Commit

Permalink
In Centos7 CONDA_DIR is different from Ubuntu - /usr/local
Browse files Browse the repository at this point in the history
  • Loading branch information
@oshadura
oshadura committed Dec 13, 2023
1 parent 9494d18 commit c167f63
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 99 deletions.
46 changes: 17 additions & 29 deletions docker/Dockerfile.cc-analysis-centos7
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,21 @@ ARG XCACHE_HOST="red-xcache1.unl.edu"


# Configure environment
ENV CONDA_DIR /usr/local
ENV CERT_DIR $CERT_DIR
ENV XCACHE_HOST $XCACHE_HOST
ENV BEARER_TOKEN_FILE $BEARER_TOKEN_FILE
ENV SEC_TOKEN_SYSTEM_DIRECTORY $SEC_TOKEN_SYSTEM_DIRECTORY
ENV CONDA_DIR /opt/conda
ENV SHELL /bin/bash
ENV NB_USER $NB_USER
ENV USER $NB_USER
ENV NB_UID $NB_UID
ENV NB_GID $NB_GID
ENV HOME /home/$NB_USER
ENV PATH "/opt/conda/bin/:$PATH"
ENV PATH "${CONDA_DIR}/bin/:$PATH"
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US.UTF-8

# Install all OS dependencies for notebook server that starts but lacks all
# features (e.g., download as all possible file formats)
Expand Down Expand Up @@ -146,29 +149,13 @@ RUN cd /tmp && \
ENV XRD_PLUGINCONFDIR="${CONDA_DIR}/etc/xrootd/client.plugins.d/"
ENV XRD_PLUGIN="${CONDA_DIR}/lib/libXrdClXcachePlugin-5.so"

# REMOVE THIS BLOCK AFTER TEST:
# ------- xrootd-authz-plugin -------------------------------
#RUN cd /tmp && \
# # ------- xrdcl-authz-plugin -------------------------------
# git clone https://github.com/bbockelm/xrdcl-authz-plugin.git && \
# cd xrdcl-authz-plugin && \
# mkdir build && \
# cd build && \
# cmake /tmp/xrdcl-authz-plugin -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} && \
# make && \
# make install && \
# ln -s ${CONDA_DIR}/lib/libXrdClAuthzPlugin-5.so ${CONDA_DIR}/lib/libXrdClAuthzPlugin.so

#ENV XRD_PLUGINCONFDIR="${CONDA_DIR}/etc/xrootd/client.plugins.d/"
#ENV XRD_PLUGIN="${CONDA_DIR}/lib/libXrdClAuthzPlugin.so"

RUN chmod 755 /etc/grid-security/certificates
COPY certs/hcc-flatiron.pem /etc/grid-security/certificates/
RUN ln -s /etc/grid-security/certificates/hcc-flatiron.pem /etc/grid-security/certificates/80d1fda9.0

# TODO: RETEST IF WE STILL NEED THIS
ENV LD_LIBRARY_PATH="/opt/conda/lib/:$LD_LIBRARY_PATH"
ENV PATH="/opt/conda/bin/:$PATH"
ENV LD_LIBRARY_PATH="${CONDA_DIR}/lib/:$LD_LIBRARY_PATH"
ENV PATH="${CONDA_DIR}/bin/:$PATH"

USER root
# Setup supervisord files
Expand All @@ -181,8 +168,8 @@ RUN groupadd -r condor && \

# FIXME: merge PRs open in distributed.git (oshadura)
# Distributed: we need to install patched version of distributed version
COPY dask/distributed /opt/conda/lib/python3.9/site-packages/distributed
RUN cd /opt/conda/lib/python3.9/site-packages/distributed && \
COPY dask/distributed ${CONDA_DIR}/lib/python3.10/site-packages/distributed
RUN cd ${CONDA_DIR}/lib/python3.10/site-packages/distributed && \
patch -p2 < 0001-Patch-from-bbockelman-adaptive-scaling.patch && \
patch -p2 < 0002-Allow-scheduler-to-preserve-worker-hostnames.patch
# && patch -p2 < 0003-Activate-patch.patch
Expand All @@ -191,22 +178,23 @@ RUN cd /opt/conda/lib/python3.9/site-packages/distributed && \

# FIXME: we have a wrong path, let's make a link.
# cms-jovyan@jupyter-oksana-2eshadura-40cern-2ech:~$ echo $PATH
# /opt/conda/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games
RUN rm -rf /opt/conda/condabin && ln -s /opt/conda/bin /opt/conda/condabin
# ${CONDA_DIR}/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games
RUN rm -rf ${CONDA_DIR}/condabin && ln -s ${CONDA_DIR}/bin ${CONDA_DIR}/condabin

# Cleanup
RUN rm -rf /tmp/* \
&& rm -rf $HOME/.cache/.pip/* \
&& find /opt/conda/ -type f,l -name '*.a' -delete \
&& find /opt/conda/ -type f,l -name '*.pyc' -delete \
&& find /opt/conda/ -type f,l -name '*.js.map' -delete \
&& rm -rf /opt/conda/pkgs
&& mamba clean --all -f -y \
&& find ${CONDA_DIR} -type f -name '*.a' -delete \
&& find ${CONDA_DIR} -type f -name '*.pyc' -delete \
&& find ${CONDA_DIR} -type f -name '*.js.map' -delete \
&& (find ${CONDA_DIR}/lib/python3.10/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete || echo "no bokeh static files to cleanup") \
&& rm -rf ${CONDA_DIR}/pkgs

# Mkdir CVMFS mount directory. Mount done upstream in configs. Harmless
# if not utilized, crucial if CVMFS is desired
RUN mkdir /cvmfs


# Prepare HTCondor spec. environment and execute dask-worker command
# If we using this container as a sidecar, we don't setup any HTCondor spec. environment
# nor and execute dask-worker command
Expand Down
121 changes: 51 additions & 70 deletions docker/Dockerfile.cc-base-centos7
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,14 @@ ARG WORKER_IMAGE="${REGISTRY}/${PROJECT}/cc-analysis-ubuntu"
ARG GITHUB_ACTIONS="false"

# Configure environment
ENV CONDA_DIR /opt/conda
ENV CONDA_DIR /usr/local
ENV SHELL /bin/bash
ENV NB_USER $NB_USER
ENV USER $NB_USER
ENV NB_UID $NB_UID
ENV NB_GID $NB_GID
ENV HOME /home/$NB_USER
ENV PATH "/opt/conda/bin/:$PATH"
ENV PATH "${CONDA_DIR}/bin/:$PATH"
ENV DASK_ROOT_CONFIG $DASK_ROOT_CONFIG
ENV LABEXTENTION_CLUSTER $LABEXTENTION_CLUSTER
ENV LABEXTENTION_FACTORY_CLASS $LABEXTENTION_FACTORY_CLASS
Expand Down Expand Up @@ -136,14 +136,20 @@ RUN mkdir "/home/${NB_USER}/work" && \
# files across image layers when the permissions change
RUN mamba install --quiet --yes \
-c conda-forge \
'notebook' \
'jupyterhub' \
'jupyterlab' \
'notebook'==6.5.3 \
'jupyterhub'==3.1.1 \
'jupyterlab'==3.6.1 \
jupyterlab_widgets==3.0.7 \
jupyterlab-git==0.41.0 \
dask=2023.11.0 \
dask_labextension==6.2.0 \
jupyter_client==8.1.0 \
jupyterlab_widgets \
jupyterlab-git \
dask_labextension && \
nodejs=14 \
ipywidgets==8.0.6 && \
mamba clean \
--all \
--force-pkgs-dirs \
--yes && \
npm cache clean --force && \
jupyter notebook --generate-config && \
Expand All @@ -167,36 +173,6 @@ RUN sed -re "s/c.NotebookApp/c.ServerApp/g" \
/etc/jupyter/jupyter_notebook_config.py > /etc/jupyter/jupyter_server_config.py && \
fix-permissions /etc/jupyter/

# Install Jupyter Notebook, Lab, and Hub
# Generate a notebook server config
# Cleanup temporary files
# Correct permissions
# Do all this in a single RUN command to avoid duplicating all of the
# files across image layers when the permissions change
RUN mamba install --quiet --yes \
-c conda-forge \
'notebook'==6.5.3 \
'jupyterhub'==3.1.1 \
'jupyterlab'==3.6.1 \
jupyterlab_widgets==3.0.7 \
jupyterlab-git==0.41.0 \
dask=2023.11.0 \
dask_labextension==6.2.0 \
jupyter_client==8.1.0 \
jupyterlab_widgets \
nodejs=14 \
ipywidgets==8.0.6 && \
mamba clean \
--all \
--force-pkgs-dirs \
--yes && \
npm cache clean --force && \
jupyter notebook --generate-config && \
jupyter lab clean && \
rm -rf "/home/${NB_USER}/.cache/yarn"
#fix-permissions "${CONDA_DIR}" && \
#fix-permissions "/home/${NB_USER}"

# Enable the serverextensions that do not use the conf.d approach and
# build JupyterLab.
RUN jupyter serverextension enable dask_labextension jupyterlab_git && \
Expand All @@ -214,17 +190,6 @@ RUN mkdir -p ${DASK_ROOT_CONFIG} && chown -R "${NB_USER}:${NB_GID}" ${DASK_ROOT
mkdir -p /var/lib/condor && \
mkdir -p /etc/condor/config.d

# Setup HTCondor user/group and change group for user $NB_USER
# Fix error (submitting jobs as user/group 0 (root) is not allowed for security reasons) and
# it configured from kubernetes side and updated in docker container to match it
RUN groupadd -r condor && \
useradd -r -g condor -d /var/lib/condor -s /sbin/nologin condor

# Fix permissions for Dask/Ceph config files
RUN chown -R "${NB_USER}:${NB_GID}" ${DASK_ROOT_CONFIG}/*.yaml
ENV LD_LIBRARY_PATH="/opt/conda/lib/:$LD_LIBRARY_PATH"
ENV PATH="/opt/conda/bin/:$PATH"

USER ${NB_UID}

RUN mamba install --yes \
Expand All @@ -247,6 +212,40 @@ RUN mamba install --yes \
--force-pkgs-dirs \
--yes

USER ${NB_UID}
# Dask, Labextention and coffea-casa setup
COPY dask/dask.yaml dask/labextension.yaml ${DASK_ROOT_CONFIG}/

USER root
# Add HTCondor configuration files that not needed to be edited
COPY condor/condor_config /etc/condor/
COPY condor/config.d /etc/condor/config.d/

RUN curl -L https://github.com/opensciencegrid/osg-vo-config/archive/refs/heads/master.tar.gz | \
tar -xz --strip-components=1 --directory=/etc/grid-security --wildcards */vomses */vomsdir && \
mv /etc/grid-security/vomses /etc

# Setup HTCondor user/group and change group for user $NB_USER
# Fix error (submitting jobs as user/group 0 (root) is not allowed for security reasons) and
# it configured from kubernetes side and updated in docker container to match it
RUN groupadd -r condor && \
useradd -r -g condor -d /var/lib/condor -s /sbin/nologin condor

# Fix permissions for Dask/Ceph config files
RUN chown -R "${NB_USER}:${NB_GID}" ${DASK_ROOT_CONFIG}/*.yaml
ENV LD_LIBRARY_PATH="${CONDA_DIR}/lib/:$LD_LIBRARY_PATH"
ENV PATH="${CONDA_DIR}/bin/:$PATH"

# FIXME: we have a wrong path, let's make a link.
# cms-jovyan@jupyter-oksana-2eshadura-40cern-2ech:~$ echo $PATH
# ${CONDA_DIR}/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games
RUN rm -rf ${CONDA_DIR}/condabin && ln -s ${CONDA_DIR}/bin ${CONDA_DIR}/condabin

# Mkdir CVMFS mount directory. Mount done upstream in configs. Harmless
# if not utilized, crucial if CVMFS is desired
RUN mkdir /cvmfs

USER ${NB_USER}
RUN pip install --no-cache-dir \
correctionlib \
funcx \
Expand Down Expand Up @@ -289,44 +288,27 @@ COPY dask/jobqueue-coffea-casa.yaml dask/dask_tls.yaml ${DASK_ROOT_CONFIG}/

USER root
# Distributed: we need to install patched version of distributed version
COPY dask/distributed ${CONDA_DIR}/lib/python3.9/site-packages/distributed
RUN cd ${CONDA_DIR}/lib/python3.9/site-packages/distributed && \
COPY dask/distributed ${CONDA_DIR}/lib/python3.10/site-packages/distributed
RUN cd ${CONDA_DIR}/lib/python3.10/site-packages/distributed && \
patch -p2 < 0001-Patch-from-bbockelman-adaptive-scaling.patch && \
patch -p2 < 0002-Allow-scheduler-to-preserve-worker-hostnames.patch
# && patch -p2 < 0003-Activate-patch.patch
# && patch -p2 < 0004-Add-possibility-to-setup-external_adress-for-schedul.patch
# && patch -p2 < 0005-Add-nanny-patch.patch


# Cleanup
RUN rm -rf /tmp/* \
&& rm -rf $HOME/.cache/.pip/* \
&& mamba clean --all -f -y \
&& jupyter lab clean \
&& jlpm cache clean \
&& npm cache clean --force \
&& find ${CONDA_DIR} -type f -name '*.a' -delete \
&& find ${CONDA_DIR} -type f -name '*.pyc' -delete \
&& find ${CONDA_DIR} -type f -name '*.js.map' -delete \
&& (find ${CONDA_DIR}/lib/python*/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete || echo "no bokeh static files to cleanup") \
&& (find ${CONDA_DIR}/lib/python3.10/site-packages/bokeh/server/static -type f,l -name '*.js' -not -name '*.min.js' -delete || echo "no bokeh static files to cleanup") \
&& rm -rf ${CONDA_DIR}/pkgs

# Fix permissions for Dask files
RUN chown -R "${NB_USER}:${NB_GID}" ${DASK_ROOT_CONFIG}/*.yaml
# xcache setup
#ENV XRD_PLUGINCONFDIR="${CONDA_DIR}/etc/xrootd/client.plugins.d/"
ENV LD_LIBRARY_PATH="${CONDA_DIR}/lib/:$LD_LIBRARY_PATH"
#ENV XRD_PLUGIN="${CONDA_DIR}/lib/libXrdClAuthzPlugin.so"
ENV PATH="${CONDA_DIR}/bin/:$PATH"

# FIXME: we have a wrong path, let's make a link.
# cms-jovyan@jupyter-oksana-2eshadura-40cern-2ech:~$ echo $PATH
# /opt/conda/condabin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games
RUN rm -rf ${CONDA_DIR}/condabin && ln -s ${CONDA_DIR}/bin ${CONDA_DIR}/condabin

# Mkdir CVMFS mount directory. Mount done upstream in configs. Harmless
# if not utilized, crucial if CVMFS is desired
RUN mkdir /cvmfs

# FIXME: add better layering for preparation of env
ADD prepare-env/prepare-env-cc.sh /usr/local/bin/prepare-env.sh
RUN chmod ugo+x /usr/local/bin/prepare-env.sh
Expand All @@ -340,4 +322,3 @@ ENTRYPOINT ["tini", "-g", "--", "/usr/local/bin/prepare-env.sh"]
# Use bash login shell for entrypoint in order
# to automatically source user's .bashrc
CMD ["start-notebook.sh"]

0 comments on commit c167f63

Please sign in to comment.