Skip to content

Commit

Permalink
ci(fix): update scripts for ci workflows (#3)
Browse files Browse the repository at this point in the history
Description
Updated some of the workflow use dscripts

Motivation and Context
Improve CI

How Has This Been Tested?
Not

What process can a PR reviewer use to test or verify this change?
Not yet

<!-- Checklist -->
<!-- 1. Is the title of your PR in the form that would make nice release
notes? The title, excluding the conventional commit
tag, will be included exactly as is in the CHANGELOG, so please think
about it carefully. -->


Breaking Changes
---

- [x] None
- [ ] Requires data directory on base node to be deleted
- [ ] Requires hard fork
- [ ] Other - Please specify

<!-- Does this include a breaking change? If so, include this line as a
footer -->
<!-- BREAKING CHANGE: Description what the user should do, e.g. delete a
database, resync the chain -->
  • Loading branch information
leet4tari authored Jul 2, 2024
1 parent 88e365a commit 9435066
Show file tree
Hide file tree
Showing 10 changed files with 347 additions and 125 deletions.
14 changes: 5 additions & 9 deletions .github/workflows/build_binaries.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,15 @@
"rust": "stable",
"target": "aarch64-unknown-linux-gnu",
"cross": true,
"flags": "--features libtor --workspace --exclude minotari_mining_helper_ffi --exclude tari_integration_tests",
"build_metric": true
"build_enabled": true,
"best_effort": true
},
{
"name": "linux-riscv64",
"runs-on": "ubuntu-latest",
"rust": "stable",
"target": "riscv64gc-unknown-linux-gnu",
"cross": true,
"flags": "--workspace --exclude minotari_mining_helper_ffi --exclude tari_integration_tests",
"build_enabled": true,
"best_effort": true
},
Expand All @@ -46,18 +45,15 @@
"rust": "stable",
"target": "x86_64-pc-windows-msvc",
"cross": false,
"features": "safe",
"flags": "--workspace --exclude tari_libtor"
"features": "safe"
},
{
"name": "windows-arm64",
"runs-on": "windows-latest",
"rust": "stable",
"target": "aarch64-pc-windows-msvc",
"cross": false,
"features": "safe",
"target_bins": "minotari_node, minotari_console_wallet, minotari_merge_mining_proxy, minotari_miner",
"flags": "--workspace --exclude tari_libtor",
"build_enabled": false
"build_enabled": true,
"best_effort": true
}
]
125 changes: 9 additions & 116 deletions .github/workflows/build_binaries.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,12 @@ name: Build Matrix of Binaries
default: "development-tag"

env:
TS_FILENAME: "tari_suite"
TS_FILENAME: "sha_p2pool"
TS_BUNDLE_ID_BASE: "com.tarilabs"
TS_SIG_FN: "sha256-unsigned.txt"
## Must be a JSon string
TS_FILES: '["minotari_node","minotari_console_wallet","minotari_miner","minotari_merge_mining_proxy"]'
TS_FEATURES: "default, safe"
TS_LIBRARIES: "minotari_mining_helper_ffi"
TARI_NETWORK_DIR: testnet
TS_FILES: '["sha_p2pool"]'
TS_FEATURES: "default"
toolchain: nightly-2024-03-01
matrix-json-file: ".github/workflows/build_binaries.json"
CARGO_HTTP_MULTIPLEXING: false
Expand Down Expand Up @@ -93,7 +91,6 @@ jobs:
needs: matrix-prep
continue-on-error: ${{ matrix.builds.best_effort || false }}
outputs:
TARI_NETWORK_DIR: ${{ steps.set-tari-network.outputs.TARI_NETWORK_DIR }}
TARI_VERSION: ${{ steps.set-tari-vars.outputs.TARI_VERSION }}
VSHA_SHORT: ${{ steps.set-tari-vars.outputs.VSHA_SHORT }}
strategy:
Expand All @@ -105,26 +102,6 @@ jobs:
steps:
- name: Checkout source code
uses: actions/checkout@v4
with:
submodules: recursive

- name: Declare TestNet for tags
id: set-tari-network
# Don't forget to comment out the below if, when force testing with GHA_NETWORK
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
env:
GHA_NETWORK: ${{ github.ref_name }}
# GHA_NETWORK: "v1.0.0-rc.4"
shell: bash
run: |
source buildtools/multinet_envs.sh ${{ env.GHA_NETWORK }}
echo ${TARI_NETWORK}
echo ${TARI_TARGET_NETWORK}
echo ${TARI_NETWORK_DIR}
echo "TARI_NETWORK=${TARI_NETWORK}" >> $GITHUB_ENV
echo "TARI_TARGET_NETWORK=${TARI_TARGET_NETWORK}" >> $GITHUB_ENV
echo "TARI_NETWORK_DIR=${TARI_NETWORK_DIR}" >> $GITHUB_ENV
echo "TARI_NETWORK_DIR=${TARI_NETWORK_DIR}" >> $GITHUB_OUTPUT

- name: Declare Global Variables 4 GHA ${{ github.event_name }}
id: set-tari-vars
Expand All @@ -136,7 +113,7 @@ jobs:
echo "VSHA_SHORT=${VSHA_SHORT}" >> $GITHUB_OUTPUT
TARI_VERSION=$(awk -F ' = ' '$1 ~ /^version/ \
{ gsub(/["]/, "", $2); printf("%s",$2) }' \
"$GITHUB_WORKSPACE/applications/minotari_node/Cargo.toml")
"$GITHUB_WORKSPACE/Cargo.toml")
echo "TARI_VERSION=${TARI_VERSION}" >> $GITHUB_ENV
echo "TARI_VERSION=${TARI_VERSION}" >> $GITHUB_OUTPUT
if [[ "${{ matrix.builds.features }}" == "" ]]; then
Expand All @@ -155,17 +132,6 @@ jobs:
TARGET_BINS+="--bin ${BIN_FILE} "
done
echo "TARGET_BINS=${TARGET_BINS}" >> $GITHUB_ENV
TARGET_LIBS=""
if [[ "${{ matrix.builds.target_libs }}" == "" ]]; then
ARRAY_LIBS=( $(echo ${TS_LIBRARIES} | tr ', ' '\n') )
else
ARRAY_LIBS=( $(echo "${{ matrix.builds.target_libs }}" | tr ', ' '\n') )
fi
for LIB_FILE in "${ARRAY_LIBS[@]}"; do
echo "Adding ${LIB_FILE} to library Builds"
TARGET_LIBS+="--package ${LIB_FILE} "
done
echo "TARGET_LIBS=${TARGET_LIBS}" >> $GITHUB_ENV
TARI_BUILD_ISA_CPU=${{ matrix.builds.target }}
# Strip unknown part
TARI_BUILD_ISA_CPU=${TARI_BUILD_ISA_CPU//-unknown-linux-gnu}
Expand Down Expand Up @@ -203,8 +169,9 @@ jobs:
- name: Install macOS dependencies
if: startsWith(runner.os,'macOS')
run: |
# openssl, cmake and autoconf already installed
brew install zip coreutils automake protobuf
# Already installed items
# brew install openssl cmake autoconf zip
brew install coreutils automake protobuf
rustup target add ${{ matrix.builds.target }}
- name: Install Windows dependencies
Expand Down Expand Up @@ -312,14 +279,6 @@ jobs:
${{ env.TARGET_BINS }} \
${{ matrix.builds.flags }} --locked
- name: Build release libraries
shell: bash
run: |
${{ env.CARGO }} build ${{ env.CARGO_OPTIONS }} \
--target ${{ matrix.builds.target }} \
--lib ${{ env.TARGET_LIBS }} \
${{ matrix.builds.flags }} --locked
- name: Copy binaries to folder for archiving
shell: bash
run: |
Expand Down Expand Up @@ -354,41 +313,8 @@ jobs:
cp -vf "${GITHUB_WORKSPACE}/target/${{ matrix.builds.target }}/release/${FILE}${LIB_EXT}" .
fi
done
if [ -f "${GITHUB_WORKSPACE}/applications/minotari_node/${PLATFORM_SPECIFIC_DIR}/runtime/start_tor${SHELL_EXT}" ]; then
cp -vf "${GITHUB_WORKSPACE}/applications/minotari_node/${PLATFORM_SPECIFIC_DIR}/runtime/start_tor${SHELL_EXT}" .
fi
ls -alhtR ${{ env.MTS_SOURCE }}
- name: Build minotari_node with metrics too
if: ${{ matrix.builds.build_metric }}
shell: bash
run: |
${{ env.CARGO }} build ${{ env.CARGO_OPTIONS }} \
--target ${{ matrix.builds.target }} \
--features "${{ env.BUILD_FEATURES }}, metrics" \
--bin minotari_node \
${{ matrix.builds.flags }} --locked
cp -vf "$GITHUB_WORKSPACE/target/${{ matrix.builds.target }}/release/minotari_node${TS_EXT}" \
"${{ env.MTS_SOURCE }}/minotari_node-metrics${TS_EXT}"
- name: Build targeted miners
# if: ${{ ( startsWith(github.ref, 'refs/tags/v') ) && ( matrix.builds.miner_cpu_targets != '' ) }}
if: ${{ matrix.builds.miner_cpu_targets != '' }}
shell: bash
run: |
ARRAY_TARGETS=( $(echo "${{ matrix.builds.miner_cpu_targets }}" | tr ', ' '\n') )
for CPU_TARGET in "${ARRAY_TARGETS[@]}"; do
echo "Target CPU ${CPU_TARGET} for miner"
export RUSTFLAGS="-C target-cpu=${CPU_TARGET}"
${{ env.CARGO }} build ${{ env.CARGO_OPTIONS }} \
--target ${{ matrix.builds.target }} \
--features "${{ env.BUILD_FEATURES }}" \
--bin minotari_miner \
${{ matrix.builds.flags }} --locked
cp -vf "$GITHUB_WORKSPACE/target/${{ matrix.builds.target }}/release/minotari_miner" \
"${{ env.MTS_SOURCE }}/minotari_miner-${CPU_TARGET}"
done
- name: Pre/unsigned OSX Artifact upload for Archive
# Debug
if: ${{ false }}
Expand Down Expand Up @@ -505,6 +431,7 @@ jobs:

- name: Build the Windows installer
if: startsWith(runner.os,'Windows')
continue-on-error: true
shell: cmd
run: |
cd buildtools
Expand All @@ -519,6 +446,7 @@ jobs:
- name: Artifact upload for Windows installer
if: startsWith(runner.os,'Windows')
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: "${{ env.TS_FILENAME }}_windows_installer"
Expand Down Expand Up @@ -549,36 +477,6 @@ jobs:
name: ${{ env.TS_FILENAME }}_archive-${{ matrix.builds.name }}
path: "${{ github.workspace }}${{ env.TS_DIST }}/${{ env.BINFILE }}.zip*"

- name: Prep diag-utils archive for upload
continue-on-error: true
shell: bash
run: |
mkdir -p "${{ env.MTS_SOURCE }}-diag-utils"
cd "${{ env.MTS_SOURCE }}-diag-utils"
# Find RandomX built tools for testing
find "$GITHUB_WORKSPACE/target/${{ matrix.builds.target }}/release/" \
-name "randomx-*${{ env.TS_EXT}}" -type f -perm -+x -exec cp -vf {} . \;
echo "Compute diag utils shasum"
${SHARUN} * \
>> "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.sha256"
cat "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.sha256"
echo "Checksum verification for diag utils is "
${SHARUN} --check "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.sha256"
7z a "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip" *
echo "Compute diag utils archive shasum"
${SHARUN} "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip" \
>> "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip.sha256"
cat "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip.sha256"
echo "Checksum verification for diag utils archive is "
${SHARUN} --check "${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}.zip.sha256"
- name: Artifact upload for diag-utils
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: ${{ env.TS_FILENAME }}_archive-diag-utils-${{ matrix.builds.name }}
path: "${{ github.workspace }}${{ env.TS_DIST }}-diag-utils/*.zip*"

macOS-universal-assemble:
name: macOS universal assemble
needs: builds
Expand Down Expand Up @@ -779,7 +677,6 @@ jobs:
needs: builds

env:
TARI_NETWORK_DIR: ${{ needs.builds.outputs.TARI_NETWORK_DIR }}
TARI_VERSION: ${{ needs.builds.outputs.TARI_VERSION }}

permissions:
Expand Down Expand Up @@ -837,10 +734,6 @@ jobs:
ls -alhtR
echo "Clean up"
# Bash check if file with wildcards, does not work as expected
# if [ -f ${{ env.TS_FILENAME }}*diag-utils* ] ; then
if ls ${{ env.TS_FILENAME }}*diag-utils* > /dev/null 2>&1 ; then
rm -fv ${{ env.TS_FILENAME }}*diag-utils*
fi
echo "Folder setup"
if ls ${{ env.TS_FILENAME }}*linux* > /dev/null 2>&1 ; then
mkdir -p "linux/${{ env.TARI_NETWORK_DIR }}/"
Expand Down
Empty file added .license.ignore
Empty file.
59 changes: 59 additions & 0 deletions scripts/cross_compile_tooling.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
#
# Move all cross-compiling steps into a single sourced script
#

set -e

printenv

if [ "${TARGETARCH}" = "arm64" ] ; then
platform_env=aarch64
export BUILD_TARGET="${platform_env}-unknown-linux-gnu/"
export RUST_TARGET="--target=${platform_env}-unknown-linux-gnu"
#export ARCH=${ARCH:-generic}
export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=${platform_env}-linux-gnu-gcc
export CC_aarch64_unknown_linux_gnu=${platform_env}-linux-gnu-gcc
export CXX_aarch64_unknown_linux_gnu=${platform_env}-linux-gnu-g++
export BINDGEN_EXTRA_CLANG_ARGS="--sysroot /usr/${platform_env}-linux-gnu/include/"
#export RUSTFLAGS="-C target_cpu=$ARCH"
#export ROARING_ARCH=$ARCH
rustup target add ${platform_env}-unknown-linux-gnu
rustup toolchain install stable-${platform_env}-unknown-linux-gnu --force-non-host

# Check for Debian
if [ -f "/etc/debian_version" ] ; then
dpkg --add-architecture ${TARGETARCH}
apt-get update || true
apt-get install -y pkg-config libssl-dev:${TARGETARCH} gcc-${platform_env}-linux-gnu g++-${platform_env}-linux-gnu
export AARCH64_UNKNOWN_LINUX_GNU_OPENSSL_INCLUDE_DIR=/usr/include/${platform_env}-linux-gnu/openssl/
export PKG_CONFIG_ALLOW_CROSS=1
fi

elif [ "${TARGETARCH}" = "amd64" ] ; then
platform_env=x86_64
platform_env_alt=x86-64
export BUILD_TARGET="${platform_env}-unknown-linux-gnu/"
export RUST_TARGET="--target=${platform_env}-unknown-linux-gnu"
# https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html
#export ARCH=${ARCH:-x86-64}
export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=${platform_env}-linux-gnu-gcc
export CC_x86_64_unknown_linux_gnu=${platform_env_alt}-linux-gnu-gcc
export CXX_x86_64_unknown_linux_gnu=${platform_env_alt}-linux-gnu-g++
export BINDGEN_EXTRA_CLANG_ARGS="--sysroot /usr/${platform_env}-linux-gnu/include/"
rustup target add ${platform_env}-unknown-linux-gnu
rustup toolchain install stable-${platform_env}-unknown-linux-gnu --force-non-host

# Check for Debian
if [ -f "/etc/debian_version" ] ; then
dpkg --add-architecture ${TARGETARCH}
apt-get update
apt-get install -y pkg-config libssl-dev:${TARGETARCH} gcc-${platform_env_alt}-linux-gnu g++-${platform_env_alt}-linux-gnu
export X86_64_UNKNOWN_LINUX_GNU_OPENSSL_INCLUDE_DIR=/usr/include/${platform_env}-linux-gnu/openssl/
export PKG_CONFIG_ALLOW_CROSS=1
fi

else
echo "Need to source [ ${0##*/} ] with env TARGETARCH set to either arm64 or amd64"
exit 1
fi
Loading

0 comments on commit 9435066

Please sign in to comment.