diff --git a/tools/fetcher/Earthfile b/tools/fetcher/Earthfile new file mode 100644 index 000000000..226b1605e --- /dev/null +++ b/tools/fetcher/Earthfile @@ -0,0 +1,55 @@ +VERSION 0.7 +FROM golang:1.20-alpine3.18 + +# cspell: words onsi ldflags extldflags + +fmt: + DO ../../earthly/go+FMT --src="go.mod go.sum cmd pkg" + +lint: + DO ../../earthly/go+LINT --src="go.mod go.sum cmd pkg" + +deps: + WORKDIR /work + + RUN apk add --no-cache gcc musl-dev + DO ../../earthly/go+DEPS + +src: + FROM +deps + + COPY --dir cmd pkg . + +check: + FROM +src + + BUILD +fmt + BUILD +lint + +build: + FROM +src + + ENV CGO_ENABLED=0 + RUN go build -ldflags="-extldflags=-static" -o bin/fetcher cmd/main.go + + SAVE ARTIFACT bin/fetcher fetcher + +test: + FROM +build + + RUN ginkgo ./... + +release: + FROM +build + + SAVE ARTIFACT bin/fetcher fetcher + +publish: + FROM debian:bookworm-slim + WORKDIR /workspace + ARG tag=latest + + COPY +build/fetcher /usr/local/bin/fetcher + + ENTRYPOINT ["/usr/local/bin/fetcher"] + SAVE IMAGE --push ci-fetcher:${tag} \ No newline at end of file diff --git a/tools/fetcher/cmd/main.go b/tools/fetcher/cmd/main.go new file mode 100644 index 000000000..a0f8e1fe0 --- /dev/null +++ b/tools/fetcher/cmd/main.go @@ -0,0 +1,91 @@ +package main + +// cspell: words alecthomas afero sess tfstate + +import ( + "fmt" + "os" + + "github.com/alecthomas/kong" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/input-output-hk/catalyst-ci/tools/fetcher/pkg" + s "github.com/input-output-hk/catalyst-ci/tools/fetcher/pkg/store" +) + +// TagTemplate is a template for generating image tags. +type TagTemplate struct { + Hash string + Timestamp string + Version string +} + +type CLI struct { + Backend string `enum:"s3" short:"s" help:"The object store backend to use." default:"s3"` + Bucket string `short:"b" help:"The object store bucket to fetch the artifact from." required:"true"` + + Archive archiveCmd `cmd:"" help:"Fetch jormungandr blockchain archives."` + Artifact artifactCmd `cmd:"" help:"Fetch jormungandr or vit-ss artifacts."` +} + +type archiveCmd struct { + Environment string `short:"e" help:"environment to fetch the archive from" required:"true"` + ID string `short:"i" help:"id of the archive to fetch"` + Path string `help:"path to store the archive" arg:"" type:"path"` +} + +func (c *archiveCmd) Run(store pkg.Store) error { + fetcher := pkg.NewArchiveFetcher(c.Environment, store) + archive, err := fetcher.Fetch(c.ID) + if err != nil { + return fmt.Errorf("failed to fetch archive: %w", err) + } + + if err := os.WriteFile(c.Path, archive, 0600); err != nil { + return fmt.Errorf("failed to save archive to disk: %w", err) + } + + return nil +} + +type artifactCmd struct { + Environment string `short:"e" help:"environment to fetch the artifact from" required:"true"` + Fund string `short:"f" help:"fund to fetch the artifact from" required:"true"` + Path string `help:"path to store the artifact" arg:"" type:"path"` + Type string `enum:"genesis,vit" short:"t" help:"type of artifact to fetch" required:"true"` + Version string `short:"v" help:"version of the artifact to fetch"` +} + +func (c *artifactCmd) Run(store pkg.Store) error { + fetcher := pkg.NewArtifactFetcher(c.Environment, c.Fund, store) + artifact, err := fetcher.Fetch(c.Type, c.Version) + if err != nil { + return fmt.Errorf("failed to fetch artifact: %w", err) + } + + if err := os.WriteFile(c.Path, artifact, 0600); err != nil { + return fmt.Errorf("failed to save artifact to disk: %w", err) + } + + return nil +} + +func main() { + cli := &CLI{} + ctx := kong.Parse(cli) + + var store pkg.Store + switch cli.Backend { + case "s3": + sess := session.Must(session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + })) + store = s.NewS3Store(cli.Bucket, sess) + default: + ctx.Fatalf("unknown backend: %s", cli.Backend) + } + + ctx.BindTo(store, (*pkg.Store)(nil)) + err := ctx.Run() + ctx.FatalIfErrorf(err) + os.Exit(0) +} diff --git a/tools/fetcher/go.mod b/tools/fetcher/go.mod new file mode 100644 index 000000000..d0236aab5 --- /dev/null +++ b/tools/fetcher/go.mod @@ -0,0 +1,23 @@ +module github.com/input-output-hk/catalyst-ci/tools/fetcher + +go 1.20 + +require ( + github.com/alecthomas/kong v0.8.1 + github.com/aws/aws-sdk-go v1.49.15 + github.com/onsi/ginkgo/v2 v2.13.2 + github.com/onsi/gomega v1.30.0 +) + +require ( + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/tools v0.14.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/tools/fetcher/go.sum b/tools/fetcher/go.sum new file mode 100644 index 000000000..f21f28924 --- /dev/null +++ b/tools/fetcher/go.sum @@ -0,0 +1,54 @@ +github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0= +github.com/alecthomas/kong v0.8.1 h1:acZdn3m4lLRobeh3Zi2S2EpnXTd1mOL6U7xVml+vfkY= +github.com/alecthomas/kong v0.8.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= +github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= +github.com/aws/aws-sdk-go v1.49.15 h1:aH9bSV4kL4ziH0AMtuYbukGIVebXddXBL0cKZ1zj15k= +github.com/aws/aws-sdk-go v1.49.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= +github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/fetcher/pkg/archive.go b/tools/fetcher/pkg/archive.go new file mode 100644 index 000000000..b7f7e8ef3 --- /dev/null +++ b/tools/fetcher/pkg/archive.go @@ -0,0 +1,20 @@ +package pkg + +import "fmt" + +type ArchiveFetcher struct { + Environment string + Store Store +} + +func (f *ArchiveFetcher) Fetch(id string) ([]byte, error) { + key := fmt.Sprintf("%s/%s", f.Environment, id) + return f.Store.Fetch(key) +} + +func NewArchiveFetcher(environment string, store Store) ArchiveFetcher { + return ArchiveFetcher{ + Environment: environment, + Store: store, + } +} diff --git a/tools/fetcher/pkg/artifact.go b/tools/fetcher/pkg/artifact.go new file mode 100644 index 000000000..633fdfe32 --- /dev/null +++ b/tools/fetcher/pkg/artifact.go @@ -0,0 +1,38 @@ +package pkg + +import "fmt" + +// typeMap maps artifact types to their corresponding file names. +var TypeMap = map[string]string{ + "genesis": "block0.bin", + "vit": "database.sqlite3", +} + +// ArtifactFetcher is used to fetch artifacts from a store. +type ArtifactFetcher struct { + Environment string + Fund string + Store Store +} + +// Fetch fetches the given artifact from the store. +// If id is empty, it will use the base artifact filename with no version suffix. +func (f *ArtifactFetcher) Fetch(artifactType string, version string) ([]byte, error) { + var key string + if version == "" { + key = fmt.Sprintf("%s/%s/%s", f.Environment, f.Fund, TypeMap[artifactType]) + } else { + key = fmt.Sprintf("%s/%s/%s-%s", f.Environment, f.Fund, TypeMap[artifactType], version) + } + + return f.Store.Fetch(key) +} + +// NewArtifactFetcher creates a new ArtifactFetcher. +func NewArtifactFetcher(environment, fund string, store Store) ArtifactFetcher { + return ArtifactFetcher{ + Environment: environment, + Fund: fund, + Store: store, + } +} diff --git a/tools/fetcher/pkg/store.go b/tools/fetcher/pkg/store.go new file mode 100644 index 000000000..163fe8efa --- /dev/null +++ b/tools/fetcher/pkg/store.go @@ -0,0 +1,7 @@ +package pkg + +// Store is an interface for fetching data from an archive/artifact store. +type Store interface { + // Fetch fetches the given key from the store. + Fetch(key string) ([]byte, error) +} diff --git a/tools/fetcher/pkg/store/aws.go b/tools/fetcher/pkg/store/aws.go new file mode 100644 index 000000000..510c93eb0 --- /dev/null +++ b/tools/fetcher/pkg/store/aws.go @@ -0,0 +1,46 @@ +package store + +import ( + "fmt" + "io" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" +) + +// S3Store is an implementation of the Store interface for fetching data from +// AWS S3. +type S3Store struct { + Bucket string + S3 s3iface.S3API + Session *session.Session +} + +func (s *S3Store) Fetch(key string) ([]byte, error) { + resp, err := s.S3.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(s.Bucket), + Key: aws.String(key), + }) + + if err != nil { + return nil, fmt.Errorf("failed to fetch S3 object: %w", err) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read S3 object: %w", err) + } + + return data, nil +} + +// NewS3Store creates a new S3Store. +func NewS3Store(bucket string, session *session.Session) *S3Store { + return &S3Store{ + Bucket: bucket, + Session: session, + S3: s3.New(session), + } +} diff --git a/tools/fetcher/pkg/store/aws_test.go b/tools/fetcher/pkg/store/aws_test.go new file mode 100644 index 000000000..0b09e1358 --- /dev/null +++ b/tools/fetcher/pkg/store/aws_test.go @@ -0,0 +1,92 @@ +package store_test + +import ( + "bytes" + "fmt" + "io" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" + s "github.com/input-output-hk/catalyst-ci/tools/fetcher/pkg/store" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type mockS3Client struct { + s3iface.S3API + Bucket string + Key string + Response []byte + Err error +} + +func (m *mockS3Client) GetObject( + input *s3.GetObjectInput, +) (*s3.GetObjectOutput, error) { + if m.Err != nil { + return nil, m.Err + } + + m.Bucket = *input.Bucket + m.Key = *input.Key + + return &s3.GetObjectOutput{ + Body: io.NopCloser(bytes.NewReader(m.Response)), + }, nil +} + +var _ = Describe("Aws", func() { + Describe("Fetch", func() { + var store s.S3Store + + When("A valid response is returned", func() { + BeforeEach(func() { + store = s.S3Store{ + Bucket: "test-bucket", + Session: session.Must(session.NewSession()), + S3: &mockS3Client{ + Response: []byte("test"), + Err: nil, + }, + } + }) + + It("should use the correct bucket", func() { + _, err := store.Fetch("test-key") + Expect(err).ToNot(HaveOccurred()) + Expect(store.S3.(*mockS3Client).Bucket).To(Equal("test-bucket")) + }) + + It("should use the correct key", func() { + _, err := store.Fetch("test-key") + Expect(err).ToNot(HaveOccurred()) + Expect(store.S3.(*mockS3Client).Key).To(Equal("test-key")) + }) + + It("should return the correct object data", func() { + data, err := store.Fetch("test-key") + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(Equal([]byte("test"))) + }) + }) + + When("An error response is returned", func() { + BeforeEach(func() { + store = s.S3Store{ + Bucket: "test-bucket", + Session: session.Must(session.NewSession()), + S3: &mockS3Client{ + Response: []byte("test"), + Err: fmt.Errorf("test error"), + }, + } + }) + + It("should return an error", func() { + _, err := store.Fetch("test-key") + Expect(err).To(HaveOccurred()) + }) + }) + }) +}) diff --git a/tools/fetcher/pkg/store/store_suite_test.go b/tools/fetcher/pkg/store/store_suite_test.go new file mode 100644 index 000000000..e79bb6093 --- /dev/null +++ b/tools/fetcher/pkg/store/store_suite_test.go @@ -0,0 +1,13 @@ +package store_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestStore(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Store Suite") +}