Skip to content

Commit 8a8ff60

Browse files
lindsayadlindsayad
committed
Add mfem and conduit submodules, add mfem to moose-dev, update moose-dev
- Add mfem install to moose-dev container - Update moose-dev - separate skip submodules
1 parent c308a33 commit 8a8ff60

File tree

7 files changed

+280
-20
lines changed

7 files changed

+280
-20
lines changed

‎.gitmodules‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,3 +47,11 @@
4747
path = modules/fluid_properties/contrib/carbon_dioxide
4848
url = https://github.com/idaholab/carbon_dioxide.git
4949
update = none
50+
[submodule "framework/contrib/mfem"]
51+
path = framework/contrib/mfem
52+
url = ../../mfem/mfem.git
53+
update = none
54+
[submodule "framework/contrib/conduit"]
55+
path = framework/contrib/conduit
56+
url = https://github.com/LLNL/conduit.git
57+
update = none

‎apptainer/moose-dev.def‎

Lines changed: 76 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,31 +7,52 @@
77

88
{#- Optional jinja arguments -#}
99
{#- CONDA_ALT_CHANNEL: An alternate channel to add to conda -#}
10+
{#- CONDUIT_GIT_SHA: The git SHA to use for conduit -#}
11+
{#- CONDUIT_GIT_REMOTE: The git remote to use to get conduit -#}
12+
{#- CONDUIT_OPTIONS: Options to pass to conduit build -#}
13+
{#- MFEM_GIT_SHA: The git SHA to use for MFEM -#}
14+
{#- MFEM_GIT_REMOTE: The git remote to use to get MFEM -#}
15+
{#- MFEM_OPTIONS: Options to pass to mfem build -#}
1016
{#- MOOSE_TOOLS_VERSION: The version to pin moose-tools to (if any) -#}
1117
{#- MOOSE_JOBS: Number of jobs to pass to the builds -#}
12-
{#- WITH_LIBTORCH: The libtorch distribution to use (cpu, cu121, etc); default: cpu -#}
1318
{#- PROFILING: Set to anything to build for profiling (add gperftools) -#}
19+
{#- WITH_LIBTORCH: The libtorch distribution to use (cpu, cu121, etc); default: cpu -#}
20+
{#- WITH_MFEM: Whether or not to build mfem (default: 1 to build, 0 to not build) -#}
1421

1522
{#- Set WITH_LIBTORCH default if not set -#}
1623
{%- if WITH_LIBTORCH is not defined %}
1724
{%- set WITH_LIBTORCH = 'cpu' %}
1825
{%- endif %}
26+
{#- Set WITH_MFEM default if not set -#}
27+
{%- if WITH_MFEM is not defined %}
28+
{%- set WITH_MFEM = '1' %}
29+
{%- endif %}
1930

2031
{#- The within-container build directory to use -#}
2132
{%- set ROOT_BUILD_DIR = '/root/build' -%}
2233

2334
{#- Location to install miniforge3 -#}
2435
{%- set MOOSE_MINIFORGE = '/opt/miniforge3' -%}
2536

26-
{#- The installation location for wasp -#}
27-
{%- set WASP_DIR = '/opt/wasp' -%}
37+
{#- Name of the environment to create in conda -#}
38+
{%- set MOOSE_CONDA_ENV_NAME = 'moose' -%}
39+
2840
{#- The installation location for gperftools -#}
2941
{%- set GPERF_DIR = '/opt/gperftools' -%}
3042
{#- The installation location for pprof -#}
3143
{%- set PPROF_DIR = '/opt/pprof' -%}
3244
{#- The installation location for libtorch -#}
33-
{%- set LIBTORCH_DEST = '/opt' -%}
34-
45+
{%- set LIBTORCH_DIR = '/opt/libtorch' -%}
46+
{#- The script used to install conduit -#}
47+
{%- set CONDUIT_BUILD_SCRIPT = 'update_and_rebuild_conduit.sh' -%}
48+
{#- The installation location for conduit -#}
49+
{%- set CONDUIT_DIR = '/opt/conduit' -%}
50+
{#- The script used to install mfem -#}
51+
{%- set MFEM_BUILD_SCRIPT = 'update_and_rebuild_mfem.sh' -%}
52+
{#- The installation location for mfem -#}
53+
{%- set MFEM_DIR = '/opt/mfem' -%}
54+
{#- The installation location for wasp -#}
55+
{%- set WASP_DIR = '/opt/wasp' -%}
3556
{#- The script used to install wasp -#}
3657
{%- set WASP_BUILD_SCRIPT = 'update_and_rebuild_wasp.sh' -%}
3758

@@ -50,9 +71,14 @@ Fingerprints: 0CFFCAB55E806363601C442D211817B01E0911DB
5071
export WASP_DIR={{ WASP_DIR }}
5172
export PATH=${WASP_DIR}/bin:$PATH
5273

74+
{%- if WITH_MFEM %}
75+
export CONDUIT_DIR={{ CONDUIT_DIR }}
76+
export MFEM_DIR={{ MFEM_DIR }}
77+
{%- endif %}
78+
5379
{%- if WITH_LIBTORCH %}
5480
# Make libtorch visible to moose
55-
export LIBTORCH_DIR={{ LIBTORCH_DEST }}/libtorch
81+
export LIBTORCH_DIR={{ LIBTORCH_DIR }}
5682
# Adding this to not get GPU initialization errors from MPICH
5783
export MPIR_CVAR_ENABLE_GPU=0
5884
{%- endif %}
@@ -105,17 +131,9 @@ Fingerprints: 0CFFCAB55E806363601C442D211817B01E0911DB
105131
chmod 755 /opt/code-server/bin/*
106132
chmod 644 /opt/code-server/extensions/*
107133

108-
# Clone WASP
109-
cd ${ROOT_BUILD_DIR}
110-
WASP_SRC_DIR=${ROOT_BUILD_DIR}/wasp
111-
git clone ${WASP_GIT_REMOTE} ${WASP_SRC_DIR}
112-
cd ${WASP_SRC_DIR}
113-
git checkout ${WASP_GIT_SHA}
114-
git submodule update --init --recursive
115-
116134
{%- if WITH_LIBTORCH %}
117135
# Libtorch-related vars
118-
LIBTORCH_DEST={{ LIBTORCH_DEST }}
136+
LIBTORCH_DEST=$(realpath -m {{ LIBTORCH_DIR }}/..)
119137
LIBTORCH_VER=2.1.0
120138
LIBTORCH_DISTRIBUTION={{ WITH_LIBTORCH }}
121139

@@ -138,6 +156,39 @@ Fingerprints: 0CFFCAB55E806363601C442D211817B01E0911DB
138156
{%- endif %}
139157
{%- endif %}
140158

159+
{%- if WITH_MFEM %}
160+
# Load MFEM jinja variables
161+
CONDUIT_GIT_SHA={{ CONDUIT_GIT_SHA }}
162+
CONDUIT_GIT_REMOTE={{ CONDUIT_GIT_REMOTE }}
163+
CONDUIT_BUILD_SCRIPT=${ROOT_BUILD_DIR}/{{ CONDUIT_BUILD_SCRIPT }}
164+
CONDUIT_DIR={{ CONDUIT_DIR }}
165+
MFEM_BUILD_SCRIPT=${ROOT_BUILD_DIR}/{{ MFEM_BUILD_SCRIPT }}
166+
MFEM_DIR={{ MFEM_DIR }}
167+
MFEM_GIT_SHA={{ MFEM_GIT_SHA }}
168+
MFEM_GIT_REMOTE={{ MFEM_GIT_REMOTE }}
169+
170+
# Clone conduit
171+
cd ${ROOT_BUILD_DIR}
172+
git clone $CONDUIT_GIT_REMOTE
173+
CONDUIT_SRC_DIR=${ROOT_BUILD_DIR}/conduit
174+
cd ${CONDUIT_SRC_DIR}
175+
git checkout $CONDUIT_GIT_SHA
176+
git submodule update --init
177+
178+
# Clone MFEM
179+
cd ${ROOT_BUILD_DIR}
180+
git clone $MFEM_GIT_REMOTE
181+
MFEM_SRC_DIR=${ROOT_BUILD_DIR}/mfem
182+
cd ${MFEM_SRC_DIR}
183+
git checkout $MFEM_GIT_SHA
184+
185+
# Install conduit
186+
CONDUIT_DIR=${CONDUIT_DIR} CONDUIT_SRC_DIR=${CONDUIT_SRC_DIR} ${CONDUIT_BUILD_SCRIPT} {{ CONDUIT_OPTIONS }}
187+
188+
# Install MFEM
189+
MFEM_DIR=${MFEM_DIR} MFEM_SRC_DIR=${MFEM_SRC_DIR} CONDUIT_DIR=${CONDUIT_DIR} ${MFEM_BUILD_SCRIPT} {{ MFEM_OPTIONS }}
190+
{%- endif %}
191+
141192
# Install miniforge in MOOSE_MINIFORGE
142193
MINIFORGE_SCRIPT=${ROOT_BUILD_DIR}/miniforge3.sh
143194
curl -L https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh -o ${MINIFORGE_SCRIPT}
@@ -160,6 +211,14 @@ Fingerprints: 0CFFCAB55E806363601C442D211817B01E0911DB
160211
# Needed for coverage
161212
pip3 --no-cache install lcov-cobertura
162213

214+
# Clone WASP
215+
cd ${ROOT_BUILD_DIR}
216+
WASP_SRC_DIR=${ROOT_BUILD_DIR}/wasp
217+
git clone ${WASP_GIT_REMOTE} ${WASP_SRC_DIR}
218+
cd ${WASP_SRC_DIR}
219+
git checkout ${WASP_GIT_SHA}
220+
git submodule update --init --recursive
221+
163222
# Build and install wasp
164223
# This is redundant; hopefully we can use the one from conda in the future
165224
WASP_SRC_DIR=${WASP_SRC_DIR} ${WASP_BUILD_SCRIPT} -D CMAKE_INSTALL_PREFIX:STRING=${WASP_DIR}
@@ -208,6 +267,8 @@ Fingerprints: 0CFFCAB55E806363601C442D211817B01E0911DB
208267
%files
209268
{{ FILES_DIR }}/opt/code-server/bin/code-server-setup-extensions /opt/code-server/bin/code-server-setup-extensions
210269
{{ FILES_DIR }}/opt/code-server/bin/code-server-start /opt/code-server/bin/code-server-start
270+
{{ MOOSE_DIR }}/scripts/{{ MFEM_BUILD_SCRIPT }} {{ ROOT_BUILD_DIR }}/{{ MFEM_BUILD_SCRIPT }}
271+
{{ MOOSE_DIR }}/scripts/{{ CONDUIT_BUILD_SCRIPT }} {{ ROOT_BUILD_DIR }}/{{ CONDUIT_BUILD_SCRIPT }}
211272
{{ MOOSE_DIR }}/scripts/{{ WASP_BUILD_SCRIPT }} {{ ROOT_BUILD_DIR }}/{{ WASP_BUILD_SCRIPT }}
212273
{{ MOOSE_DIR }}/scripts/configure_wasp.sh {{ ROOT_BUILD_DIR }}/configure_wasp.sh
213274
{%- if WITH_LIBTORCH %}

‎framework/contrib/conduit‎

Submodule conduit added at ad86e31

‎framework/contrib/mfem‎

Submodule mfem added at e731d2d

‎scripts/apptainer_generator.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -615,11 +615,14 @@ def add_definition_vars(self, jinja_data):
615615
jinja_data[jinja_var] = source[var]
616616

617617
# Set petsc and libmesh versions
618-
need_versions = {'petsc': {'package': 'petsc', 'submodule': 'petsc'},
619-
'libmesh': {'package': 'libmesh', 'submodule': 'libmesh'},
620-
'moose-dev': {'package': 'wasp', 'submodule': 'framework/contrib/wasp'}}
621-
for library, package_info in need_versions.items():
622-
if library == self.args.library:
618+
need_versions = {'petsc': [{'package': 'petsc', 'submodule': 'petsc'}],
619+
'libmesh': [{'package': 'libmesh', 'submodule': 'libmesh'}],
620+
'moose-dev': [{'package': 'conduit', 'submodule': 'framework/contrib/conduit'},
621+
{'package': 'mfem', 'submodule': 'framework/contrib/mfem'},
622+
{'package': 'wasp', 'submodule': 'framework/contrib/wasp'}]}
623+
needed = need_versions.get(self.args.library)
624+
if needed is not None:
625+
for package_info in needed:
623626
package = package_info['package']
624627
submodule = package_info['submodule']
625628

‎scripts/update_and_rebuild_conduit.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env bash
2+
#* This file is part of the MOOSE framework
3+
#* https://www.mooseframework.org
4+
#*
5+
#* All rights reserved, see COPYRIGHT for full restrictions
6+
#* https://github.com/idaholab/moose/blob/master/COPYRIGHT
7+
#*
8+
#* Licensed under LGPL 2.1, please see LICENSE for details
9+
#* https://www.gnu.org/licenses/lgpl-2.1.html
10+
11+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
12+
13+
for i in "$@"
14+
do
15+
shift
16+
if [[ "$i" == "-h" || "$i" == "--help" ]]; then
17+
help=1;
18+
fi
19+
20+
if [ "$i" == "--skip-submodule-update" ]; then
21+
skip_sub_update=1
22+
else # Remove everything else before passing to cmake
23+
set -- "$@" "$i"
24+
fi
25+
done
26+
27+
# Display help
28+
if [ -n "$help" ]; then
29+
echo "Usage: $0 [-h | --help | --fast | --skip-submodule-update | <conduit cmake options> ]"
30+
echo
31+
echo "-h | --help Display this message and list of available libmesh options"
32+
echo "--skip-submodule-update Do not update the conduit submodule, use the current version"
33+
echo
34+
echo "Influential variables"
35+
echo "CONDUIT_SRC_DIR Path to conduit; default: ../framework/contrib/conduit from submodule"
36+
echo "CONDUIT_DIR conduit install prefix; default: ../framework/contrib/conduit/installed"
37+
exit 0
38+
fi
39+
40+
set -e
41+
42+
if [ -n "$CONDUIT_SRC_DIR" ]; then
43+
skip_sub_update=1
44+
else
45+
CONDUIT_SRC_DIR="$(realpath -m "${SCRIPT_DIR}"/../framework/contrib/conduit)"
46+
fi
47+
CONDUIT_BUILD_DIR="${CONDUIT_SRC_DIR}/build"
48+
if [ -n "$CONDUIT_DIR" ]; then
49+
echo "INFO: CONDUIT_DIR set - overriding default installed path"
50+
echo "INFO: No cleaning will be done in specified path"
51+
else
52+
CONDUIT_DIR="${CONDUIT_SRC_DIR}/installed"
53+
rm -rf "$CONDUIT_DIR"
54+
fi
55+
56+
if [ -z "$skip_sub_update" ]; then
57+
cd "${SCRIPT_DIR}/.."
58+
git submodule update --init --checkout framework/contrib/conduit
59+
cd framework/contrib/conduit
60+
git submodule update --init
61+
fi
62+
63+
# Clean up old builds
64+
rm -rf "$CONDUIT_BUILD_DIR"
65+
66+
# Build and install conduit
67+
mkdir -p "$CONDUIT_BUILD_DIR"
68+
cd "$CONDUIT_BUILD_DIR"
69+
cmake ../src -DCMAKE_INSTALL_PREFIX="$CONDUIT_DIR"
70+
make -j "${MOOSE_JOBS:-4}"
71+
make install

‎scripts/update_and_rebuild_mfem.sh‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
#!/usr/bin/env bash
2+
#* This file is part of the MOOSE framework
3+
#* https://www.mooseframework.org
4+
#*
5+
#* All rights reserved, see COPYRIGHT for full restrictions
6+
#* https://github.com/idaholab/moose/blob/master/COPYRIGHT
7+
#*
8+
#* Licensed under LGPL 2.1, please see LICENSE for details
9+
#* https://www.gnu.org/licenses/lgpl-2.1.html
10+
11+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
12+
13+
for i in "$@"
14+
do
15+
shift
16+
if [[ "$i" == "-h" || "$i" == "--help" ]]; then
17+
help=1;
18+
fi
19+
20+
if [ "$i" == "--skip-submodule-update" ]; then
21+
skip_sub_update=1
22+
else # Remove everything else before passing to cmake
23+
set -- "$@" "$i"
24+
fi
25+
done
26+
27+
# Display help
28+
if [ -n "$help" ]; then
29+
echo "Usage: $0 [-h | --help | --fast | --skip-submodule-update | <mfem cmake options> ]"
30+
echo
31+
echo "-h | --help Display this message and list of available libmesh options"
32+
echo "--skip-submodule-update Do not update the MFEM submodule, use the current version"
33+
echo
34+
echo "Influential variables"
35+
echo "CONDUIT_DIR Path to conduit; default: ../framework/contrib/conduit/installed"
36+
echo "LIBMESH_DIR Path to libmesh (for netcdf); default ../libmesh/installed"
37+
echo "MFEM_DIR MFEM install prefix; default: ../framework/contrib/mfem/installed"
38+
echo "MFEM_SRC_DIR Path to MFEM source; default: ../framework/contrib/mfem from submodule"
39+
echo "PETSC_ARCH PETSc arch; default: arch-moose if PETSC_DIR not set"
40+
echo "PETSC_DIR Path to PETSc install; default: ../petsc"
41+
exit 0
42+
fi
43+
44+
set -e
45+
46+
if [ -n "$MFEM_SRC_DIR" ]; then
47+
skip_sub_update=1
48+
else
49+
MFEM_SRC_DIR="$(realpath -m "${SCRIPT_DIR}"/../framework/contrib/mfem)"
50+
fi
51+
MFEM_BUILD_DIR="${MFEM_SRC_DIR}/build"
52+
if [ -n "$MFEM_DIR" ]; then
53+
echo "INFO: MFEM_DIR set - overriding default installed path"
54+
echo "INFO: No cleaning will be done in specified path"
55+
else
56+
MFEM_DIR="${MFEM_SRC_DIR}/installed"
57+
rm -rf "$MFEM_DIR"
58+
fi
59+
60+
if [ -n "$CONDUIT_SRC_DIR" ]; then
61+
skip_conduit_update=1
62+
else
63+
CONDUIT_SRC_DIR="$(realpath -m "${SCRIPT_DIR}"/../framework/contrib/conduit)"
64+
fi
65+
CONDUIT_BUILD_DIR="${CONDUIT_SRC_DIR}/build"
66+
67+
CONDUIT_DIR=${CONDUIT_DIR:-$(realpath -m "${SCRIPT_DIR}/../framework/contrib/conduit/installed")}
68+
LIBMESH_DIR=${LIBMESH_DIR:-$(realpath -m "${SCRIPT_DIR}/../libmesh/installed")}
69+
if [ -z "$PETSC_DIR" ]; then
70+
PETSC_DIR=$(realpath -m "${SCRIPT_DIR}/../petsc")
71+
PETSC_ARCH="arch-moose"
72+
fi
73+
74+
if [ -z "$skip_sub_update" ]; then
75+
cd "${SCRIPT_DIR}/.."
76+
git submodule update --init --checkout framework/contrib/mfem
77+
fi
78+
79+
# Build and install mfem
80+
mkdir -p "$MFEM_BUILD_DIR"
81+
cd "$MFEM_BUILD_DIR"
82+
cmake .. \
83+
-DCMAKE_POSITION_INDEPENDENT_CODE=YES \
84+
-DMFEM_USE_OPENMP=NO \
85+
-DMFEM_THREAD_SAFE=NO \
86+
-DHYPRE_DIR="$PETSC_DIR/$PETSC_ARCH" \
87+
-DMFEM_USE_MPI=YES \
88+
-DMFEM_USE_METIS=YES \
89+
-DMFEM_USE_METIS_5=YES \
90+
-DMETIS_DIR="$PETSC_DIR/$PETSC_ARCH" \
91+
-DParMETIS_DIR="$PETSC_DIR/$PETSC_ARCH" \
92+
-DMFEM_USE_SUPERLU=YES \
93+
-DSuperLUDist_DIR="$PETSC_DIR/$PETSC_ARCH" \
94+
-DBUILD_SHARED_LIBS=ON \
95+
-DHDF5_DIR="$PETSC_DIR/$PETSC_ARCH" \
96+
-DBLAS_DIR="$PETSC_DIR/$PETSC_ARCH" \
97+
-DMFEM_ENABLE_EXAMPLES=yes \
98+
-DMFEM_ENABLE_MINIAPPS=yes \
99+
-DBLAS_LIBRARIES="$PETSC_DIR//$PETSC_ARCH/lib/libfblas.a" \
100+
-DLAPACK_LIBRARIES="$PETSC_DIR//$PETSC_ARCH/lib/libflapack.a" \
101+
-DBLAS_INCLUDE_DIRS="$PETSC_DIR//$PETSC_ARCH/include" \
102+
-DLAPACK_INCLUDE_DIRS="$PETSC_DIR//$PETSC_ARCH/include" \
103+
-DCMAKE_INSTALL_PREFIX="$MFEM_DIR" \
104+
-DMFEM_USE_PETSC=YES \
105+
-DPETSC_DIR="$PETSC_DIR" \
106+
-DPETSC_ARCH="$PETSC_ARCH" \
107+
-DCMAKE_C_COMPILER=mpicc \
108+
-DMFEM_USE_NETCDF=YES \
109+
-DNETCDF_DIR="$LIBMESH_DIR" \
110+
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
111+
-DMFEM_USE_CONDUIT=YES \
112+
-DCONDUIT_DIR="$CONDUIT_DIR" \
113+
"$@"
114+
make -j "${MOOSE_JOBS:-4}"
115+
make install

0 commit comments

Comments
 (0)