Skip to content

Commit 35edae2

Browse files
MakarovS96MakarovS96
authored
Merge pull request #49 from daimor/master
new way of building
2 parents f59cad5 + 3da7575 commit 35edae2

File tree

8 files changed

+168
-194
lines changed

8 files changed

+168
-194
lines changed

‎.github/workflows/main.yml‎

Lines changed: 153 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -5,133 +5,177 @@ on:
55
branches:
66
- master
77
repository_dispatch:
8-
types:
9-
- release
8+
types: [test workflow]
9+
workflow_dispatch:
10+
inputs:
11+
ignore_cache:
12+
description: 'Ignore cache when collecting manifests'
13+
required: false
14+
default: true
15+
type: boolean
16+
schedule:
17+
# Runs "At 04:15." (see https://crontab.guru)
18+
- cron: '15 4 * * *'
19+
1020
env:
11-
orgname: intersystemsdc
12-
name: |
21+
IMAGE_REPO: containers.intersystems.com/intersystems
22+
ORGNAME: intersystemsdc
23+
IMAGE: |
1324
iris-community
1425
irishealth-community
15-
iris-ml-community
16-
irishealth-ml-community
17-
latest: latest-em
18-
version: |
26+
TAG: |
27+
latest-em
1928
latest-cd
20-
2025.1
21-
2024.2
22-
2024.3
23-
preview: latest-preview
29+
latest-preview
30+
2431
jobs:
25-
version:
32+
prepare:
2633
runs-on: ubuntu-latest
2734
outputs:
28-
name: ${{ steps.set-matrix.outputs.name }}
29-
version: ${{ steps.set-matrix.outputs.version }}
35+
images: ${{ steps.collect.outputs.images }}
36+
targets: ${{ steps.collect.outputs.targets }}
37+
do_build: ${{ steps.collect.outputs.do_build }}
3038
steps:
31-
- id: set-matrix
39+
- name: Checkout repository
40+
uses: actions/checkout@v3
41+
- uses: actions/cache/restore@v4
42+
if: ${{ github.event.inputs.ignore_cache != 'true' }}
43+
id: restore-cache
44+
with:
45+
path: manifests.txt
46+
key: manifests
47+
restore-keys: |
48+
manifests-
49+
- name: collect manifests
50+
id: collect
3251
run: |
33-
echo name=`jq -Rsc 'split("\n") | map(select(length > 0))' <<< $'${{ env.name }}' ` >> $GITHUB_OUTPUT
34-
echo version=`jq -Rsc 'split("\n") | map(select(length > 0))' <<< $'${{ env.latest }}\n${{ env.version }}\n${{ env.preview }}' ` >> $GITHUB_OUTPUT
52+
IMAGE="${{ env.IMAGE }}"
53+
TAG="${{ env.TAG }}"
54+
for baseimage in ${IMAGE[@]}; do
55+
for tag in ${TAG[@]}; do
56+
image="${{ env.IMAGE_REPO }}/${baseimage}:${tag}"
57+
echo "Inspecting $image"
58+
docker manifest inspect $image | jq ".manifests[] | ( .platform.architecture + \" ${baseimage}\" + \" ${tag}\" + \" \" + .digest )" -r
59+
done
60+
done > latest_manifests.txt
61+
cat latest_manifests.txt
62+
images=$(while IFS= read -r line; do
63+
if ! grep "$line" manifests.txt >/dev/null 2>&1; then
64+
echo $line | awk -F' ' '{printf "{\"platform\": \"linux/%s\", \"image\": \"%s\", \"tag\": \"%s\"}\n", $1,$2,$3}'
65+
fi
66+
done < "latest_manifests.txt" | jq -cs '.[] | (select(.platform == "linux/arm64" ) | .runner = "ubuntu-24.04-arm"), (select(.platform == "linux/amd64" ) | .runner = "ubuntu-latest")' | jq -cs '.' )
67+
mv latest_manifests.txt manifests.txt
68+
images=$(echo $images | jq 'map(.target=.image)' | jq -cs '.[]' )
69+
# add ml variants
70+
images=$(echo $images | jq 'map([.,(.target=(.target|sub("-";"-ml-")))] | .[])' | jq -cs '.[]' )
71+
[ ! -z "${{ vars.DOCKER_ORGNAME }}" ] && ORGNAME="${{ vars.DOCKER_ORGNAME }}"
72+
images=$(echo $images | jq "map(.orgname=\"$ORGNAME\")" -c )
73+
echo $images | jq
74+
echo images="$images" >> $GITHUB_OUTPUT
75+
([[ "$images" != "[]" ]] && echo do_build=true || echo do_build=false ) >> $GITHUB_OUTPUT
76+
targets=$(echo $images | jq '.[] | {target, tag, orgname}' | jq -cs '.[]' | sort | uniq | jq -cs '.' )
77+
echo targets="$targets" >> $GITHUB_OUTPUT
78+
- name: Upload file
79+
uses: actions/upload-artifact@v4
80+
with:
81+
name: manifests
82+
path: manifests.txt
83+
- uses: actions/cache/save@v4
84+
with:
85+
key: manifests-${{ github.run_id }}
86+
path: manifests.txt
87+
3588
build:
36-
needs:
37-
- version
89+
needs: prepare
90+
if: ${{ needs.prepare.outputs.do_build != 'false' }}
3891
strategy:
3992
fail-fast: false
4093
matrix:
41-
name: ${{ fromJSON(needs.version.outputs.name ) }}
42-
version: ${{ fromJSON(needs.version.outputs.version ) }}
43-
runs-on: ubuntu-latest
94+
include: ${{ fromJson(needs.prepare.outputs.images) }}
95+
runs-on: ${{ matrix.runner }}
4496
steps:
45-
- name: set variables
46-
id: vars
97+
- name: Checkout repository
98+
uses: actions/checkout@v3
99+
100+
- name: Pull image
101+
id: image
47102
run: |
48-
echo "base=containers.intersystems.com/intersystems/${{ matrix.name }}:${{ matrix.version }}" >> $GITHUB_OUTPUT
49-
echo "basearm=containers.intersystems.com/intersystems/${{ matrix.name }}-arm64:${{ matrix.version }}" >> $GITHUB_OUTPUT
50-
image=${{ env.orgname }}/${{ matrix.name }}
51-
[ '${{ secrets.DOCKER_ORGNAME }}' != '' ] && image=${{ secrets.DOCKER_ORGNAME }}/${{ matrix.name }}
52-
version=${{ matrix.version }}-zpm
53-
echo "image=$image" >> $GITHUB_OUTPUT
54-
echo "version=$version" >> $GITHUB_OUTPUT
55-
tags=" -t $image:$version"
56-
[ '${{ matrix.version }}' == '${{ env.latest }}' ] && tags+=" -t $image:latest"
57-
[ '${{ matrix.version }}' == '${{ env.preview }}' ] && tags+=" -t $image:preview"
58-
echo "tags=$tags" >> $GITHUB_OUTPUT
59-
tagsarm=" -t $image-arm64:$version"
60-
[ '${{ matrix.version }}' == '${{ env.latest }}' ] && tagsarm+=" -t $image-arm64:latest"
61-
[ '${{ matrix.version }}' == '${{ env.preview }}' ] && tagsarm+=" -t $image-arm64:preview"
62-
echo "tagsarm=$tagsarm" >> $GITHUB_OUTPUT
63-
- uses: actions/checkout@v3
64-
- name: Set up QEMU
65-
uses: docker/setup-qemu-action@v2
66-
- name: Set up Docker Buildx
67-
uses: docker/setup-buildx-action@v2
103+
base=${{ env.IMAGE_REPO }}/${{ matrix.image }}:${{ matrix.tag }}
104+
docker pull $base
105+
version=$(docker image inspect --format '{{index .Config.Labels "com.intersystems.platform-version"}}' $base | cut -d'.' -f1-2)
106+
echo base=$base >> $GITHUB_OUTPUT
107+
echo version=$version >> $GITHUB_OUTPUT
108+
- name: Login to Docker Hub
109+
uses: docker/login-action@v2
68110
with:
69-
platforms: linux/amd64,linux/arm64
70-
- name: pull docker image
71-
uses: nick-invision/retry@v2
72-
with:
73-
timeout_minutes: 30
74-
max_attempts: 3
75-
retry_on: timeout
76-
command: |
77-
docker pull ${{ steps.vars.outputs.base }}
78-
- name: pull arm64 docker image
79-
id: pullarm
80-
continue-on-error: true
81-
uses: nick-invision/retry@v2
82-
with:
83-
timeout_minutes: 30
84-
max_attempts: 3
85-
retry_on: timeout
86-
command: |
87-
docker pull ${{ steps.vars.outputs.basearm }}
88-
- name: docker login
89-
id: login
90-
continue-on-error: true
91-
run: docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
92-
- name: build x86 docker image
111+
username: ${{ secrets.DOCKER_USERNAME }}
112+
password: ${{ secrets.DOCKER_PASSWORD }}
113+
- name: Set up Docker Buildx
114+
uses: docker/setup-buildx-action@v3
115+
- name: Build and push by digest
93116
id: build
94-
uses: nick-invision/retry@v2
117+
uses: docker/build-push-action@v6
95118
with:
96-
timeout_minutes: 10
97-
max_attempts: 3
98-
retry_on: any
99-
command: |
100-
docker build --progress plain --no-cache --file Dockerfile-amd64 --build-arg IMAGE=${{ steps.vars.outputs.base }} --build-arg IMAGEARM=${{ steps.vars.outputs.basearm }} ${{ steps.vars.outputs.tags }} --platform linux/amd64 .
101-
- name: push to docker hub
102-
id: push
103-
if: steps.login.outcome == 'success'
119+
context: .
120+
build-args: |
121+
BASE_IMAGE=${{ env.IMAGE_REPO }}/${{ matrix.image }}:${{ matrix.tag }}
122+
ML=$(if [[ "${{ matrix.target }}" == *-ml-* ]]; then echo "true"; else echo "false"; fi)
123+
push: true
124+
tags: ${{ matrix.orgname }}/${{ matrix.target }}
125+
outputs: type=image,push-by-digest=true,name-canonical=true,push=true
126+
- name: Save digest as artifact
104127
run: |
105-
tags="${{ steps.vars.outputs.tags }}";tags=${tags// -t / }
106-
echo $tags | xargs -n1 sh -c 'docker push $0'
107-
- name: build arm64 docker image
108-
id: buildarm
109-
if: steps.pullarm.outcome == 'success'
110-
uses: nick-invision/retry@v2
128+
mkdir -p "$RUNNER_TEMP/digests"
129+
digest="${{ steps.build.outputs.digest }}"
130+
# strip "sha256:" prefix, use remainder as filename
131+
echo ${{ steps.image.outputs.version }} > "$RUNNER_TEMP/digests/${digest#sha256:}"
132+
shell: bash
133+
- name: Upload digest
134+
uses: actions/upload-artifact@v4
111135
with:
112-
timeout_minutes: 10
113-
max_attempts: 3
114-
retry_on: any
115-
command: |
116-
docker system prune -f
117-
docker build --progress plain --no-cache --file Dockerfile-arm64 --build-arg IMAGE=${{ steps.vars.outputs.base }} --build-arg IMAGEARM=${{ steps.vars.outputs.basearm }} ${{ steps.vars.outputs.tagsarm }} --platform linux/arm64 .
118-
- name: push arm64 to docker hub
119-
id: pusharm
120-
if: steps.buildarm.outcome == 'success'
121-
run: |
122-
tags="${{ steps.vars.outputs.tagsarm }}";tags=${tags// -t / }
123-
echo $tags | xargs -n1 sh -c 'docker push $0'
124-
- name: update manifest
125-
if: steps.push.outcome == 'success' && steps.pusharm.outcome == 'success'
126-
uses: nick-invision/retry@v2
136+
name: digests-${{ matrix.target }}-${{ matrix.tag }}-${{ matrix.runner }}
137+
path: ${{ runner.temp }}/digests/*
138+
if-no-files-found: error
139+
retention-days: 1
140+
merge:
141+
needs:
142+
- prepare
143+
- build
144+
strategy:
145+
fail-fast: false
146+
matrix:
147+
include: ${{ fromJson(needs.prepare.outputs.targets) }}
148+
runs-on: ubuntu-latest
149+
steps:
150+
- name: Checkout repository
151+
uses: actions/checkout@v3
152+
- name: Download digests
153+
uses: actions/download-artifact@v4
154+
with:
155+
path: ${{ runner.temp }}/digests
156+
pattern: digests-${{ matrix.target }}-${{ matrix.tag }}-*
157+
merge-multiple: true
158+
- name: Login to Docker Hub
159+
uses: docker/login-action@v2
127160
with:
128-
timeout_minutes: 10
129-
max_attempts: 3
130-
retry_on: timeout
131-
command: |
132-
tags="${{ steps.vars.outputs.tagsarm }}";tags=${tags// -t / }
133-
echo $tags | xargs -n1 bash -c 'docker manifest rm ${1//-arm64/} || true' - $1
134-
echo $tags | xargs -n1 bash -c 'docker manifest inspect ${1//-arm64/}' - $1
135-
echo $tags | xargs -n1 bash -c 'docker manifest create ${1//-arm64/} ${1//-arm64/} $1 --amend' - $1
136-
echo $tags | xargs -n1 bash -c 'docker manifest push ${1//-arm64/}' - $1
137-
echo $tags | xargs -n1 bash -c 'docker manifest inspect ${1//-arm64/}' - $1
161+
username: ${{ secrets.DOCKER_USERNAME }}
162+
password: ${{ secrets.DOCKER_PASSWORD }}
163+
- name: Set up Docker Buildx
164+
uses: docker/setup-buildx-action@v3
165+
166+
- name: Create multi-arch manifest and push
167+
working-directory: ${{ runner.temp }}/digests
168+
run: |
169+
# turn each file name (a sha256 without prefix) into IMAGE@sha256:...
170+
version=$(cat * | head -1)
171+
docker buildx imagetools create \
172+
-t ${{ matrix.orgname }}/${{ matrix.target }}:${{ matrix.tag }} \
173+
-t ${{ matrix.orgname }}/${{ matrix.target }}:${{ matrix.tag }}-zpm \
174+
-t ${{ matrix.orgname }}/${{ matrix.target }}:${version} \
175+
-t ${{ matrix.orgname }}/${{ matrix.target }}:${version}-zpm \
176+
$([[ "${{ matrix.tag }}" == "latest-em" ]] && echo -t ${{ matrix.orgname }}/${{ matrix.target }}:latest ) \
177+
$([[ "${{ matrix.tag }}" == "latest-preview" ]] && echo -t ${{ matrix.orgname }}/${{ matrix.target }}:preview ) \
178+
$(printf '${{ matrix.orgname }}/${{ matrix.target }}@sha256:%s ' *)
179+
180+
- name: Inspect final image (optional)
181+
run: docker buildx imagetools inspect ${{ matrix.orgname }}/${{ matrix.target }}:${{ matrix.tag }}

‎Dockerfile‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
FROM --platform=$BUILDPLATFORM intersystems/iris-community:2024.1-linux-${BUILDARCH}
1+
ARG BASE_IMAGE=containers.intersystems.com/intersystems/iris-community:latest-em
2+
FROM ${BASE_IMAGE} AS base
23

34
ARG IPM_INSTALLER=https://pm.community.intersystems.com/packages/zpm/latest/installer
45

@@ -11,7 +12,7 @@ RUN \
1112
iris session $ISC_PACKAGE_INSTANCENAME -U %SYS < /tmp/iris.script && \
1213
iris stop $ISC_PACKAGE_INSTANCENAME quietly
1314

14-
FROM --platform=$TARGETPLATFORM intersystems/iris-community:2024.1-linux-${TARGETARCH}
15+
FROM ${BASE_IMAGE}
1516

1617
USER root
1718

@@ -35,7 +36,10 @@ COPY --chown=${ISC_PACKAGE_MGRUSER}:${ISC_PACKAGE_IRISGROUP} iris_ipm.py /usr/ir
3536

3637
ENV PIP_BREAK_SYSTEM_PACKAGES=1
3738

39+
ARG ML=false
40+
3841
RUN pip install irissqlcli && \
42+
([ "$ML" = "true" ] && python3 -m pip install --index-url https://registry.intersystems.com/pypi/simple --no-cache-dir --target /usr/irissys/mgr/python intersystems-iris-automl matplotlib || true) && \
3943
cat /usr/irissys/lib/python/iris_ipm.py >> /usr/irissys/lib/python/iris.py
4044

4145
COPY iriscli /home/irisowner/bin/

‎Dockerfile-amd64‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎Dockerfile-arm64‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎Dockerfile-phase1‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎Dockerfile-phase2‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

‎docker-compose-beta.yml‎

Lines changed: 0 additions & 16 deletions
This file was deleted.

0 commit comments

Comments
 (0)