From fcf7805ce477b13953e70ad76eb631feb647b65e Mon Sep 17 00:00:00 2001 From: HenryNguyen5 <6404866+HenryNguyen5@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:14:47 -0400 Subject: [PATCH] CRIB 493 current chainlink develop images are broken and dont work on crib (#14790) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Reapply "CRIB 462 update go releaser dev space to work with latest changes (#1…" (#14766) This reverts commit b476a4ada268fdea6366d1032e120849b5181fc8. * Correct interpreter for x86 and add smoke test * TEST: Remove other workflows * Fix interpreter on x86 for plugins * Fix version check * TEST: Intentionally break amd64 * Revert "TEST: Intentionally break amd64" This reverts commit 1ee47c0db70508db7c9633e6f55dcee5dd348eda. * Revert "TEST: Remove other workflows" This reverts commit 6ea2f814e3d24dcbb01f2c3d173947a6eb98a0c0. --- .../goreleaser-build-sign-publish/release.js | 152 +++++++++++++-- .goreleaser.develop.yaml | 25 ++- .goreleaser.devspace.yaml | 179 ++++++++++-------- .goreleaser.production.yaml | 25 ++- flake.lock | 53 +++++- flake.nix | 25 ++- nix-darwin-shell-hook.sh | 30 +++ shell.nix | 19 +- tools/bin/goreleaser_utils | 33 +++- tools/goreleaser-config/gen_config.go | 45 ++++- tools/goreleaser-config/main.go | 2 +- 11 files changed, 452 insertions(+), 136 deletions(-) create mode 100755 nix-darwin-shell-hook.sh diff --git a/.github/actions/goreleaser-build-sign-publish/release.js b/.github/actions/goreleaser-build-sign-publish/release.js index 565b03ee7f4..340c74af40c 100755 --- a/.github/actions/goreleaser-build-sign-publish/release.js +++ b/.github/actions/goreleaser-build-sign-publish/release.js @@ -1,25 +1,146 @@ #!/usr/bin/env node const { execSync } = require("child_process"); +const fs = require("fs"); +const path = require("path"); function main() { - const goreleaserConfig = mustGetEnv("GORELEASER_CONFIG"); - const releaseType = mustGetEnv("RELEASE_TYPE"); - const command = constructGoreleaserCommand(releaseType, goreleaserConfig); - - if (process.env.DRY_RUN) { - console.log(`Generated command: ${command}`); - console.log("Dry run enabled. Exiting without executing the command."); - return; + const args = process.argv.slice(2); + const useExistingDist = args.includes("--use-existing-dist"); + const chainlinkVersion = getVersion(); + + if (!useExistingDist) { + const goreleaserConfig = mustGetEnv("GORELEASER_CONFIG"); + const releaseType = mustGetEnv("RELEASE_TYPE"); + const command = constructGoreleaserCommand( + releaseType, + chainlinkVersion, + goreleaserConfig + ); + + if (process.env.DRY_RUN) { + console.log(`Generated command: ${command}`); + console.log("Dry run enabled. Exiting without executing the command."); + return; + } else { + console.log(`Executing command: ${command}`); + execSync(command, { stdio: "inherit" }); + } } else { - console.log(`Executing command: ${command}`); - execSync(command, { stdio: "inherit" }); + console.log( + "Skipping Goreleaser command execution as '--use-existing-dist' is set." + ); + } + + const artifactsJsonPath = findArtifactsJson(); + const dockerImages = extractDockerImages(artifactsJsonPath); + const repoSha = execSync("git rev-parse HEAD", { encoding: "utf-8" }).trim(); + + const results = dockerImages.map((image) => { + try { + console.log(`Checking version for image: ${image}, expected version: ${chainlinkVersion}, expected SHA: ${repoSha}`); + const versionOutput = execSync(`docker run --rm ${image} --version`, { + encoding: "utf-8", + }); + console.log(`Output from image ${image}: ${versionOutput}`); + + const cleanedOutput = versionOutput.replace("chainlink version ", "").trim(); + const [version, sha] = cleanedOutput.split("@"); + if (!version || !sha) { + throw new Error("Version or SHA not found in output."); + } + + if (sha.trim() !== repoSha) { + throw new Error(`SHA mismatch: Expected ${repoSha}, got ${sha.trim()}`); + } + if (version.trim() !== chainlinkVersion) { + throw new Error( + `Version mismatch: Expected ${chainlinkVersion}, got ${version.trim()}` + ); + } + + return { image, success: true, message: "Version check passed." }; + } catch (error) { + return { image, success: false, message: error.message }; + } + }); + + printSummary(results); + if (results.some((result) => !result.success)) { + process.exit(1); } } -main(); +function printSummary(results) { + const passed = results.filter((result) => result.success); + const failed = results.filter((result) => !result.success); + + console.log("\nSummary:"); + console.log(`Total images checked: ${results.length}`); + console.log(`Passed: ${passed.length}`); + console.log(`Failed: ${failed.length}`); + + if (passed.length > 0) { + console.log("\nPassed images:"); + passed.forEach((result) => + console.log(`${result.image}:\n${result.message}`) + ); + } + + if (failed.length > 0) { + console.log("\nFailed images:"); + failed.forEach((result) => + console.log(`${result.image}:\n${result.message}`) + ); + } +} + +function findArtifactsJson() { + const distDir = path.resolve(process.cwd(), "dist"); + const files = []; + + function findJsonFiles(dir) { + const items = fs.readdirSync(dir, { withFileTypes: true }); + for (const item of items) { + const fullPath = path.join(dir, item.name); + if (item.isDirectory()) { + findJsonFiles(fullPath); + } else if (item.isFile() && item.name === "artifacts.json") { + files.push(fullPath); + } + } + } + + findJsonFiles(distDir); + + if (files.length === 0) { + console.error("Error: No artifacts.json found in /dist."); + process.exit(1); + } else if (files.length > 1) { + console.error("Error: Multiple artifacts.json files found."); + process.exit(1); + } + + return files[0]; +} + +function extractDockerImages(artifactsJsonPath) { + console.log(`Reading artifacts.json from: ${artifactsJsonPath}`); + const artifactsJson = JSON.parse(fs.readFileSync(artifactsJsonPath, "utf-8")); + + const dockerImages = artifactsJson + .filter((artifact) => artifact.type === "Docker Image") + .map((artifact) => artifact.name); + + if (dockerImages.length === 0) { + console.error("Error: No Docker images found in artifacts.json."); + process.exit(1); + } -function constructGoreleaserCommand(releaseType, goreleaserConfig) { - const version = getVersion(); + console.log(`Found Docker images:\n - ${dockerImages.join("\n - ")}`); + return dockerImages; +} + +function constructGoreleaserCommand(releaseType, version, goreleaserConfig) { const flags = []; checkReleaseType(releaseType); @@ -59,6 +180,7 @@ function checkReleaseType(releaseType) { console.error( `Error: Invalid release type: ${releaseType}. Must be one of: ${validReleaseTypesStr}` ); + process.exit(1); } } @@ -74,7 +196,7 @@ function mustGetEnv(key) { function getVersion() { try { - const pkgPath = process.cwd() + "/package.json"; + const pkgPath = path.resolve(process.cwd(), "package.json"); console.log("Looking for chainlink version in package.json at: ", pkgPath); const packageJson = require(pkgPath); if (!packageJson.version) { @@ -91,3 +213,5 @@ function getVersion() { process.exit(1); } } + +main(); diff --git a/.goreleaser.develop.yaml b/.goreleaser.develop.yaml index 1e054df7c45..c633e22fe62 100644 --- a/.goreleaser.develop.yaml +++ b/.goreleaser.develop.yaml @@ -3,6 +3,7 @@ project_name: chainlink env: - IMG_PRE={{ if index .Env "IMAGE_PREFIX" }}{{ .Env.IMAGE_PREFIX }}{{ else }}localhost:5001{{ end }} - IMG_TAG={{ if index .Env "IMAGE_TAG" }}{{ .Env.IMAGE_TAG }}{{ else }}develop{{ end }} + - CGO_ENABLED=1 - VERSION={{ if index .Env "CHAINLINK_VERSION" }}{{ .Env.CHAINLINK_VERSION }}{{ else }}v0.0.0-local{{ end }} release: disable: "true" @@ -16,8 +17,14 @@ builds: no_unique_dist_dir: "true" ldflags: - -s -w -r=$ORIGIN/libs - - -X github.com/smartcontractkit/chainlink/v2/core/static.Version={{ .Env.VERSION }} - -X github.com/smartcontractkit/chainlink/v2/core/static.Sha={{ .FullCommit }} + - |- + -extldflags "-Wl,--dynamic-linker={{ if contains .Runtime.Goarch "amd64" -}} + /lib64/ld-linux-x86-64.so.2 + {{- else if contains .Runtime.Goarch "arm64" -}} + /lib/ld-linux-aarch64.so.1 + {{- end }}" + - -X github.com/smartcontractkit/chainlink/v2/core/static.Version={{ .Env.VERSION }} flags: - -trimpath - -buildmode=pie @@ -49,8 +56,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-amd64-chainlink-plugins goos: linux @@ -78,8 +85,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-arm64-chainlink goos: linux @@ -101,8 +108,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-arm64-chainlink-plugins goos: linux @@ -129,8 +136,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-amd64-ccip goos: linux @@ -155,8 +162,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-amd64-ccip-plugins goos: linux @@ -186,8 +193,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-arm64-ccip goos: linux @@ -211,8 +218,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-arm64-ccip-plugins goos: linux @@ -241,8 +248,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx docker_manifests: - id: tagged-chainlink diff --git a/.goreleaser.devspace.yaml b/.goreleaser.devspace.yaml index d0167b34cda..13027590a2b 100644 --- a/.goreleaser.devspace.yaml +++ b/.goreleaser.devspace.yaml @@ -1,89 +1,102 @@ -project_name: chainlink-devspace - version: 2 - +project_name: chainlink env: - - ZIG_EXEC={{ if index .Env "ZIG_EXEC" }}{{ .Env.ZIG_EXEC }}{{ else }}zig{{ end }} - - IMAGE_LABEL_DESCRIPTION="node of the decentralized oracle network, bridging on and off-chain computation" - - IMAGE_LABEL_LICENSES="MIT" - - IMAGE_LABEL_SOURCE="https://github.com/smartcontractkit/{{ .ProjectName }}" - -before: - hooks: - - go mod tidy - - ./tools/bin/goreleaser_utils before_hook - -# See https://goreleaser.com/customization/build/ + - IMG_PRE={{ if index .Env "IMAGE_PREFIX" }}{{ .Env.IMAGE_PREFIX }}{{ else }}localhost:5001{{ end }} + - IMG_TAG={{ if index .Env "IMAGE_TAG" }}{{ .Env.IMAGE_TAG }}{{ else }}develop{{ end }} + - CGO_ENABLED=1 +release: + disable: "true" builds: - - binary: chainlink - id: linux-amd64 - goos: - - linux - goarch: - - amd64 - hooks: - post: ./tools/bin/goreleaser_utils build_post_hook {{ dir .Path }} {{ .Os }} {{ .Arch }} - env: - - CGO_ENABLED=1 - - CC=$ZIG_EXEC cc -target x86_64-linux-gnu -Wno-error=unused-command-line-argument - - CCX=$ZIG_EXEC c++ -target x86_64-linux-gnu -Wno-error=unused-command-line-argument - flags: - - -trimpath - - -buildmode=pie - ldflags: - - -s -w -r=$ORIGIN/libs - - -X github.com/smartcontractkit/chainlink/v2/core/static.Version={{ .Version }} - - -X github.com/smartcontractkit/chainlink/v2/core/static.Sha={{ .FullCommit }} - -# See https://goreleaser.com/customization/docker/ + - targets: + - go_first_class + binary: chainlink + hooks: + post: + - cmd: ./tools/bin/goreleaser_utils build_post_hook {{ dir .Path }} + no_unique_dist_dir: "true" + ldflags: + - -s -w -r=$ORIGIN/libs + - -X github.com/smartcontractkit/chainlink/v2/core/static.Sha={{ .FullCommit }} + - |- + -extldflags "-Wl,--dynamic-linker={{ if contains .Runtime.Goarch "amd64" -}} + /lib64/ld-linux-x86-64.so.2 + {{- else if contains .Runtime.Goarch "arm64" -}} + /lib/ld-linux-aarch64.so.1 + {{- end }}" + - -X github.com/smartcontractkit/chainlink/v2/core/static.Version={{ .Version }} + flags: + - -trimpath + - -buildmode=pie +archives: + - format: binary +snapshot: + version_template: v0.0.0-{{ .Runtime.Goarch }}-{{ .Now.Format "2006-01-02-15-04-05Z" }} +checksum: + name_template: checksums.txt dockers: - - id: linux-amd64 - dockerfile: core/chainlink.goreleaser.Dockerfile - use: buildx - goos: linux - goarch: amd64 - extra_files: - - tmp/linux_amd64/libs - - tmp/linux_amd64/plugins - - tools/bin/ldd_fix - build_flag_templates: - - "--platform=linux/amd64" - - "--pull" - - "--build-arg=CHAINLINK_USER=chainlink" - - "--build-arg=COMMIT_SHA={{ .FullCommit }}" - - "--build-arg=CL_MEDIAN_CMD=chainlink-feeds" - - "--build-arg=CL_MERCURY_CMD=chainlink-mercury" - - "--build-arg=CL_SOLANA_CMD=chainlink-solana" - - "--build-arg=CL_STARKNET_CMD=chainlink-starknet" - - "--label=org.opencontainers.image.created={{ .Date }}" - - "--label=org.opencontainers.image.description={{ .Env.IMAGE_LABEL_DESCRIPTION }}" - - "--label=org.opencontainers.image.licenses={{ .Env.IMAGE_LABEL_LICENSES }}" - - "--label=org.opencontainers.image.revision={{ .FullCommit }}" - - "--label=org.opencontainers.image.source={{ .Env.IMAGE_LABEL_SOURCE }}" - - "--label=org.opencontainers.image.title={{ .ProjectName }}" - - "--label=org.opencontainers.image.version={{ .Version }}" - - "--label=org.opencontainers.image.url={{ .Env.IMAGE_LABEL_SOURCE }}" - image_templates: - - "{{ .Env.IMAGE }}" - -# See https://goreleaser.com/customization/docker_manifest/ + - id: linux-amd64 + goos: linux + goarch: amd64 + dockerfile: core/chainlink.goreleaser.Dockerfile + image_templates: + - '{{ .Env.IMAGE }}' + extra_files: + - tmp/libs + - tmp/plugins + build_flag_templates: + - --platform=linux/amd64 + - --pull + - --build-arg=CHAINLINK_USER=chainlink + - --build-arg=COMMIT_SHA={{ .FullCommit }} + - --build-arg=CL_MEDIAN_CMD=chainlink-feeds + - --build-arg=CL_MERCURY_CMD=chainlink-mercury + - --build-arg=CL_SOLANA_CMD=chainlink-solana + - --build-arg=CL_STARKNET_CMD=chainlink-starknet + - --label=org.opencontainers.image.created={{ .Date }} + - --label=org.opencontainers.image.description="node of the decentralized oracle network, bridging on and off-chain computation" + - --label=org.opencontainers.image.licenses=MIT + - --label=org.opencontainers.image.revision={{ .FullCommit }} + - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.title=chainlink + - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + use: buildx + - id: linux-arm64 + goos: linux + goarch: arm64 + dockerfile: core/chainlink.goreleaser.Dockerfile + image_templates: + - '{{ .Env.IMAGE }}' + extra_files: + - tmp/libs + - tmp/plugins + build_flag_templates: + - --platform=linux/arm64 + - --pull + - --build-arg=CHAINLINK_USER=chainlink + - --build-arg=COMMIT_SHA={{ .FullCommit }} + - --build-arg=CL_MEDIAN_CMD=chainlink-feeds + - --build-arg=CL_MERCURY_CMD=chainlink-mercury + - --build-arg=CL_SOLANA_CMD=chainlink-solana + - --build-arg=CL_STARKNET_CMD=chainlink-starknet + - --label=org.opencontainers.image.created={{ .Date }} + - --label=org.opencontainers.image.description="node of the decentralized oracle network, bridging on and off-chain computation" + - --label=org.opencontainers.image.licenses=MIT + - --label=org.opencontainers.image.revision={{ .FullCommit }} + - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.title=chainlink + - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + use: buildx docker_manifests: - - name_template: "{{ .Env.IMAGE }}" - image_templates: - - "{{ .Env.IMAGE }}" - -checksum: - name_template: "checksums.txt" - -snapshot: - version_template: '{{ .Env.CHAINLINK_VERSION }}-{{ .Runtime.Goarch }}-{{ .Now.Format "2006-01-02-15-04-05Z" }}' - + - name_template: '{{ .Env.IMAGE }}' + image_templates: + - '{{ .Env.IMAGE }}' changelog: - sort: asc - filters: - exclude: - - "^docs:" - - "^test:" -# modelines, feel free to remove those if you don't want/use them: -# yaml-language-server: $schema=https://goreleaser.com/static/schema.json -# vim: set ts=2 sw=2 tw=0 fo=cnqoj + disable: "true" +before: + hooks: + - cmd: go mod tidy + - cmd: ./tools/bin/goreleaser_utils before_hook +partial: + by: target +nightly: + version_template: v0.0.0-{{ .Runtime.Goarch }}-{{ .Now.Format "2006-01-02-15-04-05Z" }} diff --git a/.goreleaser.production.yaml b/.goreleaser.production.yaml index ada9b847e74..b4e19d1b31c 100644 --- a/.goreleaser.production.yaml +++ b/.goreleaser.production.yaml @@ -3,6 +3,7 @@ project_name: chainlink env: - IMG_PRE={{ if index .Env "IMAGE_PREFIX" }}{{ .Env.IMAGE_PREFIX }}{{ else }}localhost:5001{{ end }} - IMG_TAG={{ if index .Env "IMAGE_TAG" }}{{ .Env.IMAGE_TAG }}{{ else }}develop{{ end }} + - CGO_ENABLED=1 - VERSION={{ if index .Env "CHAINLINK_VERSION" }}{{ .Env.CHAINLINK_VERSION }}{{ else }}v0.0.0-local{{ end }} release: disable: "true" @@ -16,8 +17,14 @@ builds: no_unique_dist_dir: "true" ldflags: - -s -w -r=$ORIGIN/libs - - -X github.com/smartcontractkit/chainlink/v2/core/static.Version={{ .Env.VERSION }} - -X github.com/smartcontractkit/chainlink/v2/core/static.Sha={{ .FullCommit }} + - |- + -extldflags "-Wl,--dynamic-linker={{ if contains .Runtime.Goarch "amd64" -}} + /lib64/ld-linux-x86-64.so.2 + {{- else if contains .Runtime.Goarch "arm64" -}} + /lib/ld-linux-aarch64.so.1 + {{- end }}" + - -X github.com/smartcontractkit/chainlink/v2/core/static.Version={{ .Env.VERSION }} flags: - -trimpath - -buildmode=pie @@ -50,8 +57,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-amd64-chainlink-plugins goos: linux @@ -80,8 +87,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-arm64-chainlink goos: linux @@ -104,8 +111,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-arm64-chainlink-plugins goos: linux @@ -133,8 +140,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-amd64-ccip goos: linux @@ -160,8 +167,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-amd64-ccip-plugins goos: linux @@ -192,8 +199,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-arm64-ccip goos: linux @@ -218,8 +225,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx - id: linux-arm64-ccip-plugins goos: linux @@ -249,8 +256,8 @@ dockers: - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink - --label=org.opencontainers.image.title=chainlink - - --label=org.opencontainers.image.version={{ .Env.VERSION }} - --label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink + - --label=org.opencontainers.image.version={{ .Env.VERSION }} use: buildx docker_manifests: - id: tagged-chainlink-chainlink-experimental-goreleaser diff --git a/flake.lock b/flake.lock index 77dddea4060..71af2318c95 100644 --- a/flake.lock +++ b/flake.lock @@ -40,6 +40,24 @@ "type": "github" } }, + "goreleaser-nur": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1726594821, + "narHash": "sha256-ORImH+i+zOCMOdztNDqGDbyyFRC/FKmgbX8w50TNbQY=", + "owner": "goreleaser", + "repo": "nur", + "rev": "bd2ee272ddfffbda9377a472131728e83ce2332d", + "type": "github" + }, + "original": { + "owner": "goreleaser", + "repo": "nur", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1666753130, @@ -55,6 +73,22 @@ } }, "nixpkgs_2": { + "locked": { + "lastModified": 1624561540, + "narHash": "sha256-izJ2PYZMGMsSkg+e7c9A1x3t/yOLT+qzUM6WQsc2tqo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c6a049a3d32293b24c0f894a840872cf67fd7c11", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { "locked": { "lastModified": 1725103162, "narHash": "sha256-Ym04C5+qovuQDYL/rKWSR+WESseQBbNAe5DsXNx5trY=", @@ -70,11 +104,28 @@ "type": "github" } }, + "nur": { + "locked": { + "lastModified": 1727912806, + "narHash": "sha256-LDOTTOGPaEP233gBrL8dnPGopc1lqcJFe0VB/+K/yWc=", + "owner": "nix-community", + "repo": "NUR", + "rev": "9d9bcd30fec126b08b49020b7af08bc1aad66210", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "NUR", + "type": "github" + } + }, "root": { "inputs": { "flake-utils": "flake-utils", "foundry": "foundry", - "nixpkgs": "nixpkgs_2" + "goreleaser-nur": "goreleaser-nur", + "nixpkgs": "nixpkgs_3", + "nur": "nur" } }, "systems": { diff --git a/flake.nix b/flake.nix index 7e188857ba4..0d1aac2f05d 100644 --- a/flake.nix +++ b/flake.nix @@ -6,6 +6,8 @@ foundry.url = "github:shazow/foundry.nix/monthly"; flake-utils.url = "github:numtide/flake-utils"; foundry.inputs.flake-utils.follows = "flake-utils"; + nur.url = "github:nix-community/NUR"; + goreleaser-nur.url = "github:goreleaser/nur"; }; outputs = inputs @ { @@ -13,15 +15,30 @@ nixpkgs, flake-utils, foundry, + nur, + goreleaser-nur, ... }: flake-utils.lib.eachDefaultSystem (system: let - pkgs = import nixpkgs { - inherit system; - overlays = [foundry.overlay]; - }; + isCrib = builtins.getEnv "IS_CRIB" == "true"; + pkgs = import nixpkgs { inherit system; + config = { allowUnfree = true; }; + overlays = [ + (final: prev: { + nur = import nur + { + pkgs = prev; + repoOverrides = { + goreleaser = import goreleaser-nur { pkgs = prev; }; + }; + }; + }) + foundry.overlay + ]; + }; in rec { devShell = pkgs.callPackage ./shell.nix { + isCrib = isCrib; inherit pkgs; }; formatter = pkgs.nixpkgs-fmt; diff --git a/nix-darwin-shell-hook.sh b/nix-darwin-shell-hook.sh new file mode 100755 index 00000000000..49dbfc60983 --- /dev/null +++ b/nix-darwin-shell-hook.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# This script is used to set up cross compilation to linux arm64 in a CRIB environment. +# It's loaded during the shell hook execution in shell.nix +main() { + echo "Running in CRIB environment, setting up cross compilation to linux arm64..." + PACKAGE="aarch64-unknown-linux-gnu" + + if ! command -v brew >/dev/null 2>&1; then + echo -e "\e[31mHomebrew is not installed. Please install Homebrew first: https://brew.sh/\e[0m" + exit 1 + fi + + if ! brew list --formula | grep $PACKAGE > /dev/null; then + echo -e "\e[31mThe Homebrew package $PACKAGE is not installed.\e[0m" + echo -e "\e[31mPlease install it by running: brew tap messense/macos-cross-toolchains && brew install ${PACKAGE}\e[0m" + exit 1 + fi + + export GOOS=linux + + installed_version=$(brew list --versions $PACKAGE | awk '{print $2}') + path_prefix=$(brew --prefix) + bin_path=$path_prefix/Cellar/$PACKAGE/$installed_version/bin + + export CC=$bin_path/aarch64-linux-gnu-gcc + export CXX=$bin_path/aarch64-linux-gnu-g++ +} + +main "$@" diff --git a/shell.nix b/shell.nix index ba09ebc219d..e3b187dcd96 100644 --- a/shell.nix +++ b/shell.nix @@ -1,4 +1,4 @@ -{pkgs}: +{pkgs, isCrib}: with pkgs; let go = go_1_21; postgresql = postgresql_14; @@ -18,7 +18,6 @@ in nativeBuildInputs = [ go - goreleaser postgresql python3 @@ -51,10 +50,22 @@ in pkg-config libudev-zero libusb1 + ] ++ lib.optionals isCrib [ + nur.repos.goreleaser.goreleaser-pro + patchelf ]; - LD_LIBRARY_PATH = "${stdenv.cc.cc.lib}/lib64:$LD_LIBRARY_PATH"; - GOROOT = "${go}/share/go"; + shellHook = '' + ${if !isCrib then "" else '' + if [ -z $GORELEASER_KEY ]; then + echo "GORELEASER_KEY must be set in CRIB environments. You can find it in our 1p vault under 'goreleaser-pro-license'." + exit 1 + fi + ${if stdenv.isDarwin then "source ./nix-darwin-shell-hook.sh" else ""} + ''} + ''; + + GOROOT = "${go}/share/go"; PGDATA = "db"; CL_DATABASE_URL = "postgresql://chainlink:chainlink@localhost:5432/chainlink_test?sslmode=disable"; } diff --git a/tools/bin/goreleaser_utils b/tools/bin/goreleaser_utils index a01c1654133..b4b7f124ba7 100755 --- a/tools/bin/goreleaser_utils +++ b/tools/bin/goreleaser_utils @@ -13,7 +13,38 @@ before_hook() { install_local_plugins install_remote_plugins mkdir -p "$lib_path/plugins" - cp "$(go env GOPATH)"/bin/chainlink* "$lib_path/plugins" + + # Retrieve GOPATH + GOPATH=$(go env GOPATH) + GOARCH=$(go env GOARCH) + + # Define the source directories + BIN_DIR="$GOPATH/bin" + PLUGIN_DIR="$lib_path/plugins" + + # Because we still do cross compilation in the case of + # darwin_arm64 -> linux_arm64, the plugin path will have a suffix of + # linux_arm64, rather than being suffixless on native platforms + if [ "$GOARCH" = "arm64" ]; then + if [ -d "$BIN_DIR/linux_arm64" ]; then + cp "$BIN_DIR/linux_arm64"/chainlink* "$PLUGIN_DIR" + else + cp "$BIN_DIR"/chainlink* "$PLUGIN_DIR" + fi + # Call patchelf --set-interpreter on all plugins + for plugin in "$PLUGIN_DIR"/chainlink*; do + patchelf --set-interpreter /lib/ld-linux-aarch64.so.1 "$plugin" + done + + else + cp "$BIN_DIR"/chainlink* "$PLUGIN_DIR" + + # Call patchelf --set-interpreter on all plugins + for plugin in "$PLUGIN_DIR"/chainlink*; do + patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 "$plugin" + done + fi + } install_local_plugins() { diff --git a/tools/goreleaser-config/gen_config.go b/tools/goreleaser-config/gen_config.go index 9108ec1bbd1..de0a4ef0c3c 100644 --- a/tools/goreleaser-config/gen_config.go +++ b/tools/goreleaser-config/gen_config.go @@ -13,10 +13,12 @@ var validEnvironments = []string{"devspace", "develop", "production"} func Generate(environment string) config.Project { checkEnvironments(environment) + architectures := []string{"amd64", "arm64"} + project := config.Project{ ProjectName: "chainlink", Version: 2, - Env: commonEnv(), + Env: commonEnv(environment), Before: config.Before{ Hooks: []config.Hook{ { @@ -28,7 +30,7 @@ func Generate(environment string) config.Project { }, }, Builds: builds(environment), - Dockers: dockers(environment), + Dockers: dockers(environment, architectures), DockerManifests: dockerManifests(environment), Checksum: config.Checksum{ NameTemplate: "checksums.txt", @@ -54,6 +56,11 @@ func Generate(environment string) config.Project { Disable: "true", }, } + if environment == "devspace" { + versionTemplate := `v0.0.0-{{ .Runtime.Goarch }}-{{ .Now.Format "2006-01-02-15-04-05Z" }}` + project.Snapshot = config.Snapshot{VersionTemplate: versionTemplate} + project.Nightly = config.Nightly{VersionTemplate: versionTemplate} + } // Add SBOMs if needed if environment == "production" { @@ -95,12 +102,17 @@ func checkEnvironments(environment string) { } // commonEnv returns the common environment variables used across environments. -func commonEnv() []string { - return []string{ +func commonEnv(environment string) []string { + envs := []string{ `IMG_PRE={{ if index .Env "IMAGE_PREFIX" }}{{ .Env.IMAGE_PREFIX }}{{ else }}localhost:5001{{ end }}`, `IMG_TAG={{ if index .Env "IMAGE_TAG" }}{{ .Env.IMAGE_TAG }}{{ else }}develop{{ end }}`, - `VERSION={{ if index .Env "CHAINLINK_VERSION" }}{{ .Env.CHAINLINK_VERSION }}{{ else }}v0.0.0-local{{ end }}`, + `CGO_ENABLED=1`, + } + + if environment != "devspace" { + envs = append(envs, `VERSION={{ if index .Env "CHAINLINK_VERSION" }}{{ .Env.CHAINLINK_VERSION }}{{ else }}v0.0.0-local{{ end }}`) } + return envs } // builds returns the build configurations based on the environment. @@ -122,13 +134,22 @@ func builds(environment string) []config.Build { // build creates a build configuration. func build(isDevspace bool) config.Build { + dynamicLinker := `{{ if contains .Runtime.Goarch "amd64" -}} +/lib64/ld-linux-x86-64.so.2 +{{- else if contains .Runtime.Goarch "arm64" -}} +/lib/ld-linux-aarch64.so.1 +{{- end }}` + ldflags := []string{ "-s -w -r=$ORIGIN/libs", - "-X github.com/smartcontractkit/chainlink/v2/core/static.Version={{ .Env.VERSION }}", "-X github.com/smartcontractkit/chainlink/v2/core/static.Sha={{ .FullCommit }}", + fmt.Sprintf(`-extldflags "-Wl,--dynamic-linker=%s"`, dynamicLinker), } + if isDevspace { - ldflags[2] = "-X github.com/smartcontractkit/chainlink/v2/core/static.Version={{ .Version }}" + ldflags = append(ldflags, "-X github.com/smartcontractkit/chainlink/v2/core/static.Version={{ .Version }}") + } else { + ldflags = append(ldflags, "-X github.com/smartcontractkit/chainlink/v2/core/static.Version={{ .Env.VERSION }}") } return config.Build{ @@ -148,16 +169,16 @@ func build(isDevspace bool) config.Build { } // dockers returns the docker configurations based on the environment. -func dockers(environment string) []config.Docker { +func dockers(environment string, architectures []string) []config.Docker { var dockers []config.Docker switch environment { case "devspace": dockers = []config.Docker{ docker("linux-amd64", "linux", "amd64", environment, true), + docker("linux-arm64", "linux", "arm64", environment, true), } case "develop", "production": - architectures := []string{"amd64", "arm64"} imageNames := []string{"chainlink", "ccip"} for _, imageName := range imageNames { @@ -213,9 +234,13 @@ func docker(id, goos, goarch, environment string, isDevspace bool) config.Docker `--label=org.opencontainers.image.revision={{ .FullCommit }}`, `--label=org.opencontainers.image.source=https://github.com/smartcontractkit/chainlink`, `--label=org.opencontainers.image.title=chainlink`, - `--label=org.opencontainers.image.version={{ .Env.VERSION }}`, `--label=org.opencontainers.image.url=https://github.com/smartcontractkit/chainlink`, ) + if !isDevspace { + buildFlagTemplates = append(buildFlagTemplates, + `--label=org.opencontainers.image.version={{ .Env.VERSION }}`, + ) + } dockerConfig := config.Docker{ ID: id, diff --git a/tools/goreleaser-config/main.go b/tools/goreleaser-config/main.go index 852b5b580ae..1e3e9776b14 100644 --- a/tools/goreleaser-config/main.go +++ b/tools/goreleaser-config/main.go @@ -7,7 +7,7 @@ import ( ) func main() { - environments := []string{"develop", "production"} + environments := []string{"develop", "production", "devspace"} for _, e := range environments { cfg := Generate(e) data, err := yaml.Marshal(&cfg)