Skip to content

Commit 6e61a77

Browse files
kislaykishorekislaykishore
authored
Make the Docker images architecture-agnostic so that they can run both on AMD64 as well as ARM64 architectures. (#49)
1 parent 19b0e13 commit 6e61a77

File tree

7 files changed

+27
-58
lines changed

7 files changed

+27
-58
lines changed

‎npi/Makefile

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,30 +13,28 @@
1313
# limitations under the License.
1414

1515
GCSFUSE_VERSION ?= master
16-
ARCH ?= amd64
1716
GO_VERSION=1.25.0
1817
UBUNTU_VERSION=24.04
1918
REGISTRY=us-docker.pkg.dev
2019

2120
.DEFAULT_GOAL := build
2221

23-
.PHONY: build base read-benchmark write-benchmark
22+
.PHONY: build base read-benchmark write-benchmark orbax-benchmark
2423

2524
base:
26-
docker build -f gcsfuse-perf-base.docker --build-arg GCSFUSE_VERSION=$(GCSFUSE_VERSION) --build-arg ARCH=$(ARCH) --build-arg GO_VERSION=$(GO_VERSION) --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) -t $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/gcsfuse-$(GCSFUSE_VERSION)-perf-base-$(ARCH):latest . && docker push $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/gcsfuse-$(GCSFUSE_VERSION)-perf-base-$(ARCH):latest
25+
docker buildx build --load -f gcsfuse-perf-base.docker --platform linux/amd64,linux/arm64 --build-arg GCSFUSE_VERSION=$(GCSFUSE_VERSION) --build-arg GO_VERSION=$(GO_VERSION) --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) -t $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/gcsfuse-$(GCSFUSE_VERSION)-perf-base:latest . && docker push $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/gcsfuse-$(GCSFUSE_VERSION)-perf-base:latest
2726

2827
read-benchmark: base
29-
docker build fio -f fio/read.dockerfile --build-arg GCSFUSE_VERSION=$(GCSFUSE_VERSION) --build-arg ARCH=$(ARCH) --build-arg GO_VERSION=$(GO_VERSION) --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) --build-arg REGISTRY=$(REGISTRY) -t $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/fio-read-benchmark-$(GCSFUSE_VERSION)-$(ARCH):latest && docker push $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/fio-read-benchmark-$(GCSFUSE_VERSION)-$(ARCH):latest
28+
docker buildx build --load fio -f fio/read.dockerfile --platform linux/amd64,linux/arm64 --build-arg GCSFUSE_VERSION=$(GCSFUSE_VERSION) --build-arg GO_VERSION=$(GO_VERSION) --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) --build-arg REGISTRY=$(REGISTRY) -t $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/fio-read-benchmark-$(GCSFUSE_VERSION):latest && docker push $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/fio-read-benchmark-$(GCSFUSE_VERSION):latest
3029

3130
write-benchmark: base
32-
docker build fio -f fio/write.dockerfile --build-arg GCSFUSE_VERSION=$(GCSFUSE_VERSION) --build-arg ARCH=$(ARCH) --build-arg GO_VERSION=$(GO_VERSION) --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) --build-arg REGISTRY=$(REGISTRY) -t $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/fio-write-benchmark-$(GCSFUSE_VERSION)-$(ARCH):latest && docker push $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/fio-write-benchmark-$(GCSFUSE_VERSION)-$(ARCH):latest
31+
docker buildx build --load fio -f fio/write.dockerfile --platform linux/amd64,linux/arm64 --build-arg GCSFUSE_VERSION=$(GCSFUSE_VERSION) --build-arg GO_VERSION=$(GO_VERSION) --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) --build-arg REGISTRY=$(REGISTRY) -t $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/fio-write-benchmark-$(GCSFUSE_VERSION):latest && docker push $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/fio-write-benchmark-$(GCSFUSE_VERSION):latest
3332

3433
fullsweep-benchmark: base
35-
docker build fio -f fio/fullsweep.dockerfile --build-arg GCSFUSE_VERSION=$(GCSFUSE_VERSION) --build-arg ARCH=$(ARCH) --build-arg GO_VERSION=$(GO_VERSION) --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) --build-arg REGISTRY=$(REGISTRY) -t $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/fio-fullsweep-benchmark-$(GCSFUSE_VERSION)-$(ARCH):latest && docker push $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/fio-fullsweep-benchmark-$(GCSFUSE_VERSION)-$(ARCH):latest
34+
docker buildx build --load fio -f fio/fullsweep.dockerfile --platform linux/amd64,linux/arm64 --build-arg GCSFUSE_VERSION=$(GCSFUSE_VERSION) --build-arg GO_VERSION=$(GO_VERSION) --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) --build-arg REGISTRY=$(REGISTRY) -t $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/fio-fullsweep-benchmark-$(GCSFUSE_VERSION):latest && docker push $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/fio-fullsweep-benchmark-$(GCSFUSE_VERSION):latest
3635

3736
orbax-benchmark: base
38-
docker build fio -f fio/orbax_emulated.dockerfile --build-arg GCSFUSE_VERSION=$(GCSFUSE_VERSION) --build-arg ARCH=$(ARCH) --build-arg GO_VERSION=$(GO_VERSION) --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) --build-arg REGISTRY=$(REGISTRY) -t $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/orbax-emulated-benchmark-$(GCSFUSE_VERSION)-$(ARCH):latest && docker push $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/orbax-emulated-benchmark-$(GCSFUSE_VERSION)-$(ARCH):latest
37+
docker buildx build --load fio -f fio/orbax_emulated.dockerfile --platform linux/amd64,linux/arm64 --build-arg GCSFUSE_VERSION=$(GCSFUSE_VERSION) --build-arg GO_VERSION=$(GO_VERSION) --build-arg UBUNTU_VERSION=$(UBUNTU_VERSION) --build-arg REGISTRY=$(REGISTRY) -t $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/orbax-emulated-benchmark-$(GCSFUSE_VERSION):latest && docker push $(REGISTRY)/gcs-fuse-test/gcsfuse-benchmarks/orbax-emulated-benchmark-$(GCSFUSE_VERSION):latest
3938

4039
build: read-benchmark write-benchmark fullsweep-benchmark orbax-benchmark base
4140

42-

‎npi/fio/fio.dockerfile

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
ARG UBUNTU_VERSION=24.04
22
ARG GO_VERSION=1.24.5
3-
ARG ARCH=amd64
43
ARG GCSFUSE_VERSION=master
54
ARG REGISTRY=us-docker.pkg.dev
65

7-
FROM ${REGISTRY}/gcs-fuse-test/gcsfuse-${GCSFUSE_VERSION}-perf-base-${ARCH}:latest
6+
FROM ${REGISTRY}/gcs-fuse-test/gcsfuse-${GCSFUSE_VERSION}-perf-base:latest
87
COPY run_fio_benchmark.py /run_fio_benchmark.py
98
COPY fio_benchmark_runner.py /fio_benchmark_runner.py
109
ENTRYPOINT ["/run_fio_benchmark.py"]

‎npi/fio/fullsweep.dockerfile

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
ARG UBUNTU_VERSION=24.04
22
ARG GO_VERSION=1.24.5
3-
ARG ARCH=amd64
43
ARG GCSFUSE_VERSION=master
54
ARG REGISTRY=us-docker.pkg.dev
65

7-
FROM ${REGISTRY}/gcs-fuse-test/gcsfuse-benchmarks/gcsfuse-${GCSFUSE_VERSION}-perf-base-${ARCH}:latest
6+
FROM ${REGISTRY}/gcs-fuse-test/gcsfuse-benchmarks/gcsfuse-${GCSFUSE_VERSION}-perf-base:latest
87
COPY run_fio_benchmark.py /run_fio_benchmark.py
98
COPY fio_benchmark_runner.py /fio_benchmark_runner.py
109
COPY run_fio_matrix.py /run_fio_matrix.py

‎npi/fio/orbax_emulated.dockerfile

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
ARG UBUNTU_VERSION=24.04
22
ARG GO_VERSION=1.24.5
3-
ARG ARCH=amd64
43
ARG GCSFUSE_VERSION=master
54
ARG REGISTRY=us-docker.pkg.dev
65

7-
FROM ${REGISTRY}/gcs-fuse-test/gcsfuse-benchmarks/gcsfuse-${GCSFUSE_VERSION}-perf-base-${ARCH}:latest
6+
FROM ${REGISTRY}/gcs-fuse-test/gcsfuse-benchmarks/gcsfuse-${GCSFUSE_VERSION}-perf-base:latest
87
COPY run_fio_benchmark.py /run_fio_benchmark.py
98
COPY fio_benchmark_runner.py /fio_benchmark_runner.py
109
COPY run_fio_matrix.py /run_fio_matrix.py

‎npi/fio/read.dockerfile

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
ARG UBUNTU_VERSION=24.04
22
ARG GO_VERSION=1.24.5
3-
ARG ARCH=amd64
43
ARG GCSFUSE_VERSION=master
54
ARG REGISTRY=us-docker.pkg.dev
65

7-
FROM ${REGISTRY}/gcs-fuse-test/gcsfuse-benchmarks/gcsfuse-${GCSFUSE_VERSION}-perf-base-${ARCH}:latest
6+
FROM ${REGISTRY}/gcs-fuse-test/gcsfuse-benchmarks/gcsfuse-${GCSFUSE_VERSION}-perf-base:latest
87
COPY run_fio_benchmark.py /run_fio_benchmark.py
98
COPY fio_benchmark_runner.py /fio_benchmark_runner.py
109
COPY run_fio_matrix.py /run_fio_matrix.py

‎npi/fio/write.dockerfile

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
ARG UBUNTU_VERSION=24.04
22
ARG GO_VERSION=1.24.5
3-
ARG ARCH=amd64
43
ARG GCSFUSE_VERSION=master
54
ARG REGISTRY=us-docker.pkg.dev
65

7-
FROM ${REGISTRY}/gcs-fuse-test/gcsfuse-benchmarks/gcsfuse-${GCSFUSE_VERSION}-perf-base-${ARCH}:latest
6+
FROM ${REGISTRY}/gcs-fuse-test/gcsfuse-benchmarks/gcsfuse-${GCSFUSE_VERSION}-perf-base:latest
87
COPY run_fio_benchmark.py /run_fio_benchmark.py
98
COPY fio_benchmark_runner.py /fio_benchmark_runner.py
109
COPY run_fio_matrix.py /run_fio_matrix.py

‎npi/gcsfuse-perf-base.docker

Lines changed: 16 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,22 @@
11
ARG UBUNTU_VERSION=24.04
22
ARG GO_VERSION=1.24.5
3-
ARG ARCH=amd64
43
ARG GCSFUSE_VERSION=master
54

6-
FROM ubuntu:${UBUNTU_VERSION} AS builder
7-
8-
# Re-declare ARGs to make them available inside this build stage
9-
ARG GO_VERSION
10-
ARG ARCH
5+
FROM --platform=$BUILDPLATFORM golang:${GO_VERSION} AS builder
116
ARG GCSFUSE_VERSION
12-
13-
# Set DEBIAN_FRONTEND to noninteractive to avoid prompts during package installation
14-
ENV DEBIAN_FRONTEND=noninteractive
15-
16-
# Install dependencies
17-
RUN apt-get update -y && apt-get upgrade -y && \
18-
apt-get install -y git wget make gcc
19-
20-
# Download and install Go, then add it to the PATH
21-
RUN wget "https://go.dev/dl/go${GO_VERSION}.linux-${ARCH}.tar.gz" -O go.tar.gz && \
22-
tar -C /usr/local -xzf go.tar.gz
23-
ENV PATH="/usr/local/go/bin:${PATH}"
24-
25-
# Clone GCSFuse repository, build it, and clean up apt cache
26-
RUN git clone -b ${GCSFUSE_VERSION} --depth 1 --single-branch https://github.com/GoogleCloudPlatform/gcsfuse.git && \
27-
cd gcsfuse && go build .
28-
29-
30-
FROM ubuntu:${UBUNTU_VERSION}
31-
RUN apt-get update && \
32-
apt-get upgrade -y && \
33-
apt-get install -y fuse3 python3-venv fio && \
34-
rm -rf /var/lib/apt/lists/* && \
35-
apt-get autoremove -y && \
36-
apt-get clean
37-
38-
# Create and activate virtual environment to avoid PEP 668
39-
ENV VIRTUAL_ENV=/opt/venv
40-
RUN python3 -m venv $VIRTUAL_ENV
41-
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
42-
43-
# Install Python packages into the virtual environment
44-
RUN pip install --no-cache-dir google-cloud-bigquery
45-
COPY --from=builder /gcsfuse/gcsfuse /gcsfuse/gcsfuse
7+
ARG TARGETOS
8+
ARG TARGETARCH
9+
ENV GOOS=${TARGETOS}
10+
ENV GOARCH=${TARGETARCH}
11+
WORKDIR /app
12+
RUN git clone -b ${GCSFUSE_VERSION} --depth 1 --single-branch https://github.com/GoogleCloudPlatform/gcsfuse.git
13+
RUN cd gcsfuse && go build .
14+
15+
FROM python:3.13
16+
# Install FUSE and related packages
17+
RUN apt-get update && apt-get install -y fuse3 fio \
18+
--no-install-recommends && \
19+
rm -rf /var/lib/apt/lists/* && \
20+
pip install --no-cache-dir google-cloud-bigquery
21+
COPY --from=builder /app/gcsfuse /gcsfuse/gcsfuse
4622
ENTRYPOINT ["/bin/bash"]

0 commit comments

Comments
 (0)