Skip to content

Commit c0c0acb

Browse files
gridbugsgridbugs
authored
Add monorepo benchmark (#7202)
* Add monorepo benchmark This adds a current-bench benchmark that measures the time it takes to build a large monorepo composed from opam packages. Signed-off-by: Stephen Sherratt <[email protected]>
1 parent a29a129 commit c0c0acb

File tree

7 files changed

+259
-0
lines changed

7 files changed

+259
-0
lines changed

‎.dockerignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
_build
22
_boot
3+
_opam
34
dune.exe
45
result

‎bench/monorepo/Makefile‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Intended to be used from inside docker containers built from monorepo-bench.Dockerfile
2+
RUNNER = _build/default/bench.exe
3+
DUNE_TO_BENCHMARK = /home/user/dune/_build/default/bin/main.exe
4+
5+
$(RUNNER): dune bench.ml
6+
dune build $@ --release
7+
8+
bench: $(RUNNER)
9+
$< $(DUNE_TO_BENCHMARK) build -j auto
10+
11+
clean:
12+
dune clean
13+
14+
.PHONY: bench clean

‎bench/monorepo/README.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Monorepo Bench
2+
3+
Files for building a docker image for benchmarking dune building a large
4+
monorepo composed of packages from the opam repo. Running `make bench` will
5+
build the monorepo and print out a json object with benchmark results expected
6+
to be consumed by current-bench.
7+
8+
The monorepo will be set up with opam-monorepo using the lockfile
9+
[here](https://github.com/ocaml-dune/ocaml-monorepo-benchmark/blob/main/benchmark/monorepo-bench.opam.locked)
10+
which is downloaded during `docker build`. Also during `docker build`,
11+
a a library is created called `monorepo` with
12+
[this](https://github.com/ocaml-dune/ocaml-monorepo-benchmark/blob/main/benchmark/dune)
13+
dune file listing all the libraries to include in the build.

‎bench/monorepo/bench.Dockerfile‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
# Creates a monorepo out of packages in opam and builds it with dune
2+
3+
FROM debian
4+
5+
# Enable non-free packages
6+
RUN sed -i '/^deb/ s/$/ non-free/' /etc/apt/sources.list
7+
8+
# Install tools and system dependencies of packages
9+
RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y \
10+
build-essential \
11+
sudo \
12+
pkg-config \
13+
opam \
14+
wget \
15+
autoconf \
16+
zlib1g-dev \
17+
libcairo2-dev \
18+
libcurl4-gnutls-dev \
19+
libsnmp-dev \
20+
libgmp-dev \
21+
libbluetooth-dev \
22+
cmake \
23+
libfarmhash-dev \
24+
libgl-dev \
25+
libnlopt-dev \
26+
libmpfr-dev \
27+
r-base-core \
28+
libjemalloc-dev \
29+
libsnappy-dev \
30+
libpapi-dev \
31+
libgles2 \
32+
libgles2-mesa-dev \
33+
fswatch \
34+
librdkafka-dev \
35+
google-perftools \
36+
libgoogle-perftools-dev \
37+
libglew-dev \
38+
guile-3.0-dev \
39+
portaudio19-dev \
40+
libglpk-dev \
41+
libportmidi-dev \
42+
libmpg123-dev \
43+
libgtksourceview-3.0-dev \
44+
libhidapi-dev \
45+
libfftw3-dev \
46+
libasound2-dev \
47+
libzmq3-dev \
48+
r-base-dev \
49+
libgtk2.0-dev \
50+
libsoundtouch-dev \
51+
libmp3lame-dev \
52+
libplplot-dev \
53+
libogg-dev \
54+
libavutil-dev \
55+
libavfilter-dev \
56+
libswresample-dev \
57+
libavcodec-dev \
58+
libfdk-aac-dev \
59+
libfaad2 \
60+
libsamplerate0-dev \
61+
libao-dev \
62+
liblmdb-dev \
63+
libnl-3-dev \
64+
libnl-route-3-dev \
65+
sqlite3 \
66+
libsqlite3-dev \
67+
cargo \
68+
libtool \
69+
libopenimageio-dev \
70+
libtidy-dev \
71+
libleveldb-dev \
72+
libgtkspell-dev \
73+
libtag1-dev \
74+
libsrt-openssl-dev \
75+
liblo-dev \
76+
libmad0-dev \
77+
frei0r-plugins-dev \
78+
libavdevice-dev \
79+
libfaad-dev \
80+
libglfw3-dev \
81+
protobuf-compiler \
82+
libuv1-dev \
83+
libxen-dev \
84+
libflac-dev \
85+
libpq-dev \
86+
libtheora-dev \
87+
libonig-dev \
88+
libglib2.0-dev \
89+
libgoocanvas-2.0-dev \
90+
libgtkspell3-3-dev \
91+
libpulse-dev \
92+
libdlm-dev \
93+
capnproto \
94+
libtorch-dev \
95+
libqrencode-dev \
96+
libshine-dev \
97+
libopus-dev \
98+
libspeex-dev \
99+
libvorbis-dev \
100+
libgstreamer1.0-dev \
101+
libgstreamer-plugins-base1.0-dev \
102+
liblz4-dev \
103+
liblilv-dev \
104+
libopenexr-dev \
105+
llvm \
106+
libclang-dev \
107+
libmaxminddb-dev \
108+
libsecp256k1-dev \
109+
libstring-shellquote-perl \
110+
libopenblas-dev \
111+
qt5-qmake \
112+
libqt5quick5 \
113+
qtdeclarative5-dev \
114+
libgpiod-dev \
115+
libzstd-dev \
116+
;
117+
118+
# create a non-root user
119+
RUN useradd --create-home --shell /bin/bash --gid users --groups sudo user
120+
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
121+
ENV HOME=/home/user
122+
USER user
123+
WORKDIR $HOME
124+
125+
# set up opam
126+
RUN opam init --disable-sandboxing --auto-setup
127+
128+
# make an opam switch for running benchmarks
129+
RUN opam switch create bench 4.14.1
130+
RUN opam install -y dune ocamlbuild
131+
132+
# make an opam switch for preparing the files for the benchmark
133+
RUN opam switch create prepare 4.14.1
134+
RUN opam install -y opam-monorepo ppx_sexp_conv ocamlfind ctypes ctypes-foreign re sexplib menhir camlp-streams zarith stdcompat refl
135+
136+
# Make a directory to store the monorepo benchmark project
137+
RUN mkdir -p $HOME/monorepo-bench
138+
139+
# Download the monorepo benchmark and copy files into the benchmark project
140+
ENV MONOREPO_BENCHMARK_TAG=2023-03-16.0
141+
RUN wget https://github.com/ocaml-dune/ocaml-monorepo-benchmark/archive/refs/tags/$MONOREPO_BENCHMARK_TAG.tar.gz -O ocaml-monorepo-benchmark.tar.gz && tar xf ocaml-monorepo-benchmark.tar.gz && mv ocaml-monorepo-benchmark-$MONOREPO_BENCHMARK_TAG ocaml-monorepo-benchmark
142+
WORKDIR $HOME/monorepo-bench
143+
RUN mkdir -p monorepo && cp -r $HOME/ocaml-monorepo-benchmark/benchmark/dune $HOME/ocaml-monorepo-benchmark/benchmark/monorepo.ml monorepo && cp -r $HOME/ocaml-monorepo-benchmark/benchmark/monorepo-bench.opam $HOME/ocaml-monorepo-benchmark/benchmark/monorepo-bench.opam.locked $HOME/ocaml-monorepo-benchmark/benchmark/patches .
144+
145+
# Running `opam monorepo pull` with a large package set is very likely to fail on at least
146+
# one package in a non-deterministic manner. Repeating it several times reduces the chance
147+
# that all attempts fail.
148+
RUN opam monorepo pull || opam monorepo pull || opam monorepo pull
149+
150+
# Initialize some projects' source code
151+
RUN . ~/.profile && cd duniverse/clangml && ./configure
152+
RUN cd duniverse/zelus && ./configure
153+
RUN rm -rf duniverse/magic-trace/vendor
154+
RUN cd duniverse/cpu && autoconf && autoheader && ./configure
155+
RUN cd duniverse/setcore && autoconf && autoheader && ./configure
156+
RUN cd duniverse/batsat-ocaml && ./build_rust.sh
157+
158+
# Some packages define conflicting definitions of libraries so they must be removed for the build to succeed
159+
RUN rm -r duniverse/coq-of-ocaml
160+
RUN rm -r duniverse/coq
161+
162+
# Bulid the dune binary that we'll be benchmarking
163+
RUN mkdir -p $HOME/dune
164+
WORKDIR $HOME/dune
165+
ADD --chown=user:users . .
166+
RUN . ~/.profile && dune build bin/main.exe --release
167+
168+
# Copy the remaininder of the files needed for the monorepo benchmark
169+
WORKDIR $HOME/monorepo-bench
170+
ADD --chown=user:users bench/monorepo .
171+
172+
# Apply some custom packages to some packages
173+
RUN bash -c 'for f in patches/*; do p=$(basename ${f%.diff}); patch -p1 -d duniverse/$p < $f; done'
174+
175+
# Change to the benchmarking switch to run the benchmark
176+
RUN opam switch bench

‎bench/monorepo/bench.ml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
(* Monorepo benchmark runner *)
2+
3+
(* Run a program on a list of arguments, returning the wallclock duration of
4+
the program in seconds. *)
5+
let time_run_blocking program args =
6+
let args_arr = Array.of_list (program :: args) in
7+
let timestamp_before = Unix.gettimeofday () in
8+
let child_pid =
9+
Unix.create_process program args_arr Unix.stdin Unix.stdout Unix.stderr
10+
in
11+
let got_pid, status = Unix.waitpid [] child_pid in
12+
let timestamp_after = Unix.gettimeofday () in
13+
if got_pid <> child_pid then failwith "wait returned unexpected pid";
14+
let () =
15+
match status with
16+
| Unix.WEXITED 0 -> ()
17+
| _ ->
18+
let command_string = String.concat " " (program :: args) in
19+
failwith (Printf.sprintf "`%s` did not exit successfully" command_string)
20+
in
21+
timestamp_after -. timestamp_before
22+
23+
let current_bench_json_string ~command ~wallclock_duration_secs =
24+
let command_str = String.concat " " command in
25+
Printf.sprintf
26+
{|{
27+
"results": [
28+
{
29+
"name": "running command: %s",
30+
"metrics": [
31+
{
32+
"name": "wallclock duration",
33+
"value": %f,
34+
"units": "sec"
35+
}
36+
]
37+
}
38+
]
39+
}|}
40+
command_str wallclock_duration_secs
41+
42+
let () =
43+
let argv = Array.to_list Sys.argv in
44+
let usage () = Printf.sprintf "%s <program> [<arg>, ...]" (List.hd argv) in
45+
match Array.to_list Sys.argv |> List.tl with
46+
| [] -> Printf.eprintf "Usage: %s\n" (usage ())
47+
| program :: args as command ->
48+
let wallclock_duration_secs = time_run_blocking program args in
49+
print_endline (current_bench_json_string ~command ~wallclock_duration_secs)

‎bench/monorepo/dune‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
(executable
2+
(public_name bench)
3+
(modules bench)
4+
(libraries unix))

‎bench/monorepo/dune-project‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
(lang dune 3.5)
2+
(package (name monorepo-bench))

0 commit comments

Comments
 (0)