feat: adds fetcher CLI (#144)

.config/dictionaries/project.dic

@@ -22,10 +22,12 @@ golangci
 GOLANGCI
 golines
 golint
+gomega
 GOMODCACHE
 gopls
 gosec
 graphviz
+jormungandr
 Kroki
 kubeconfig
 ldflags
@@ -57,3 +59,4 @@ uniseg
 webkitallowfullscreen
 WORKDIR
 xerrors
+zstd

cspell.json

@@ -40,6 +40,8 @@
         ".markdownlint.jsonc",
         ".markdownlint-cli2.jsonc",
         ".envrc",
-        "cspell.json"
+        "cspell.json",
+        "**/**/go.mod",
+        "**/**/go.sum",
     ]
 }

tools/fetcher/Earthfile

@@ -0,0 +1,55 @@
+VERSION 0.7
+FROM golang:1.20-alpine3.18
+
+# cspell: words onsi ldflags extldflags
+
+fmt:
+    DO ../../earthly/go+FMT --src="go.mod go.sum cmd pkg"
+
+lint:
+    DO ../../earthly/go+LINT --src="go.mod go.sum cmd pkg"
+
+deps:
+    WORKDIR /work
+
+    RUN apk add --no-cache gcc musl-dev
+    DO ../../earthly/go+DEPS
+
+src:
+    FROM +deps
+
+    COPY --dir cmd pkg .
+
+check:
+    FROM +src
+
+    BUILD +fmt
+    BUILD +lint
+
+build:
+    FROM +src
+
+    ENV CGO_ENABLED=0
+    RUN go build -ldflags="-extldflags=-static" -o bin/fetcher cmd/main.go
+
+    SAVE ARTIFACT bin/fetcher fetcher
+
+test:
+    FROM +build
+
+    RUN ginkgo ./...
+
+release:
+    FROM +build
+
+    SAVE ARTIFACT bin/fetcher fetcher
+
+publish:
+    FROM debian:bookworm-slim
+    WORKDIR /workspace
+    ARG tag=latest
+
+    COPY +build/fetcher /usr/local/bin/fetcher
+
+    ENTRYPOINT ["/usr/local/bin/fetcher"]
+    SAVE IMAGE --push ci-fetcher:${tag}

tools/fetcher/README.md

@@ -0,0 +1,44 @@
+# fetcher
+
+> A simple CLI for fetching Jormungandr archives and artifacts
+
+The `fetcher` CLI provides a simple interface for fetching Jormungandr archives and artifacts from a configured object store.
+By default, it uses AWS S3.
+The primary use of this CLI is in CI pipelines or container entrypoint scripts where archives or artifacts are needed.
+By specifying the appropriate identification parameters, the CLI will automatically download the respective files from an object
+store.
+
+## Usage
+
+The CLI uses cloud SDKs (like AWS) for authentication.
+It assumes you've configured the appropriate credentials in your environment to authenticate with the cloud provider.
+
+### Artifacts
+
+To fetch a specific Jormungandr genesis file:
+
+```shell
+# Fetch v1.0.0 of the genesis file from the fund10 dev environment
+fetcher --bucket "artifacts-bucket" artifact -e "dev" -f "fund10" -t "genesis" -v "1.0.0" ./block0.bin
+```
+
+To fetch a specific vit-ss database file:
+
+```shell
+# Fetch v1.0.0 of the vit-ss database from the fund10 dev environment
+fetcher --bucket "artifacts-bucket" artifact -e "dev" -f "fund10" -t "vit" -v "1.0.0" ./block0.bin
+```
+
+In either case, you can omit the version (`-v`) flag and the CLI will download the "default" (unversioned) artifact.
+This is offered to support backwards compatibility when using unversioned artifacts.
+
+### Archives
+
+To fetch a specific Jormungandr archive:
+
+```shell
+# Fetches an archive with the ID of f41c4470-03dd-4d3f-a325-6f784259b9c5
+fetcher --bucket "archives-bucket" archive -e "dev" -i "f41c4470-03dd-4d3f-a325-6f784259b9c5" ./artifact.tar.zstd
+```
+
+Artifact IDs can be found using the archiver API (i.e., [the dev API](https://archiver.dev.projectcatalyst.io/api/v1/archives/)).

tools/fetcher/cmd/main.go

@@ -0,0 +1,91 @@
+package main
+
+// cspell: words alecthomas afero sess tfstate
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/alecthomas/kong"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/input-output-hk/catalyst-ci/tools/fetcher/pkg"
+	s "github.com/input-output-hk/catalyst-ci/tools/fetcher/pkg/store"
+)
+
+// TagTemplate is a template for generating image tags.
+type TagTemplate struct {
+	Hash      string
+	Timestamp string
+	Version   string
+}
+
+type CLI struct {
+	Backend string `enum:"s3" short:"s" help:"The object store backend to use." default:"s3"`
+	Bucket  string `short:"b" help:"The object store bucket to fetch the artifact from." required:"true"`
+
+	Archive  archiveCmd  `cmd:"" help:"Fetch jormungandr blockchain archives."`
+	Artifact artifactCmd `cmd:"" help:"Fetch jormungandr or vit-ss artifacts."`
+}
+
+type archiveCmd struct {
+	Environment string `short:"e" help:"environment to fetch the archive from" required:"true"`
+	ID          string `short:"i" help:"id of the archive to fetch"`
+	Path        string `help:"path to store the archive" arg:"" type:"path"`
+}
+
+func (c *archiveCmd) Run(store pkg.Store) error {
+	fetcher := pkg.NewArchiveFetcher(c.Environment, store)
+	archive, err := fetcher.Fetch(c.ID)
+	if err != nil {
+		return fmt.Errorf("failed to fetch archive: %w", err)
+	}
+
+	if err := os.WriteFile(c.Path, archive, 0600); err != nil {
+		return fmt.Errorf("failed to save archive to disk: %w", err)
+	}
+
+	return nil
+}
+
+type artifactCmd struct {
+	Environment string `short:"e" help:"environment to fetch the artifact from" required:"true"`
+	Fund        string `short:"f" help:"fund to fetch the artifact from" required:"true"`
+	Path        string `help:"path to store the artifact" arg:"" type:"path"`
+	Type        string `enum:"genesis,vit" short:"t" help:"type of artifact to fetch" required:"true"`
+	Version     string `short:"v" help:"version of the artifact to fetch"`
+}
+
+func (c *artifactCmd) Run(store pkg.Store) error {
+	fetcher := pkg.NewArtifactFetcher(c.Environment, c.Fund, store)
+	artifact, err := fetcher.Fetch(c.Type, c.Version)
+	if err != nil {
+		return fmt.Errorf("failed to fetch artifact: %w", err)
+	}
+
+	if err := os.WriteFile(c.Path, artifact, 0600); err != nil {
+		return fmt.Errorf("failed to save artifact to disk: %w", err)
+	}
+
+	return nil
+}
+
+func main() {
+	cli := &CLI{}
+	ctx := kong.Parse(cli)
+
+	var store pkg.Store
+	switch cli.Backend {
+	case "s3":
+		sess := session.Must(session.NewSessionWithOptions(session.Options{
+			SharedConfigState: session.SharedConfigEnable,
+		}))
+		store = s.NewS3Store(cli.Bucket, sess)
+	default:
+		ctx.Fatalf("unknown backend: %s", cli.Backend)
+	}
+
+	ctx.BindTo(store, (*pkg.Store)(nil))
+	err := ctx.Run()
+	ctx.FatalIfErrorf(err)
+	os.Exit(0)
+}

tools/fetcher/go.mod

@@ -0,0 +1,23 @@
+module github.com/input-output-hk/catalyst-ci/tools/fetcher
+
+go 1.20
+
+require (
+	github.com/alecthomas/kong v0.8.1
+	github.com/aws/aws-sdk-go v1.49.15
+	github.com/onsi/ginkgo/v2 v2.13.2
+	github.com/onsi/gomega v1.30.0
+)
+
+require (
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

tools/fetcher/go.sum

@@ -0,0 +1,54 @@
+github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0=
+github.com/alecthomas/kong v0.8.1 h1:acZdn3m4lLRobeh3Zi2S2EpnXTd1mOL6U7xVml+vfkY=
+github.com/alecthomas/kong v0.8.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U=
+github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
+github.com/aws/aws-sdk-go v1.49.15 h1:aH9bSV4kL4ziH0AMtuYbukGIVebXddXBL0cKZ1zj15k=
+github.com/aws/aws-sdk-go v1.49.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
+github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs=
+github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM=
+github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
+github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
+golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tools/fetcher/pkg/archive.go

@@ -0,0 +1,20 @@
+package pkg
+
+import "fmt"
+
+type ArchiveFetcher struct {
+	Environment string
+	Store       Store
+}
+
+func (f *ArchiveFetcher) Fetch(id string) ([]byte, error) {
+	key := fmt.Sprintf("%s/%s", f.Environment, id)
+	return f.Store.Fetch(key)
+}
+
+func NewArchiveFetcher(environment string, store Store) ArchiveFetcher {
+	return ArchiveFetcher{
+		Environment: environment,
+		Store:       store,
+	}
+}

tools/fetcher/pkg/archive_test.go

@@ -0,0 +1,39 @@
+package pkg_test
+
+import (
+	"github.com/input-output-hk/catalyst-ci/tools/fetcher/pkg"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Archive", func() {
+	Describe("Fetch", func() {
+		When("fetching an archive", func() {
+			var store *MockStore
+			var usedKey string
+
+			BeforeEach(func() {
+				store = &MockStore{
+					FetchFunc: func(key string) ([]byte, error) {
+						usedKey = key
+						return []byte("archive"), nil
+					},
+				}
+			})
+
+			It("should use the correct key", func() {
+				fetcher := pkg.NewArchiveFetcher("test", store)
+				_, err := fetcher.Fetch("id")
+				Expect(err).ToNot(HaveOccurred())
+				Expect(usedKey).To(Equal("test/id"))
+			})
+
+			It("should return the archive", func() {
+				fetcher := pkg.NewArchiveFetcher("test", store)
+				archive, err := fetcher.Fetch("id")
+				Expect(err).ToNot(HaveOccurred())
+				Expect(archive).To(Equal([]byte("archive")))
+			})
+		})
+	})
+})