diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index 3f6edf8e8..3becdb665 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -53,6 +53,7 @@ sqlfluff subproject subprojects superfences +templating testpackage Timoni transpiling diff --git a/services/jorm-metrics-server/deployment.yml b/services/jorm-metrics-server/deployment.yml new file mode 100644 index 000000000..5aeeeb932 --- /dev/null +++ b/services/jorm-metrics-server/deployment.yml @@ -0,0 +1,4 @@ +overrides: + - app: jormungandr + path: exporter.image.tag + value: GITHUB_SHA diff --git a/tools/updater/Earthfile b/tools/updater/Earthfile index d3d3b0cfc..7e6868e1b 100644 --- a/tools/updater/Earthfile +++ b/tools/updater/Earthfile @@ -1,5 +1,5 @@ VERSION 0.7 -FROM golang:1.20-alpine3.18 +FROM golang:1.21-alpine3.18 # cspell: words onsi ldflags extldflags diff --git a/tools/updater/README.md b/tools/updater/README.md index ea2344a0f..d7b0984b4 100644 --- a/tools/updater/README.md +++ b/tools/updater/README.md @@ -1,17 +1,45 @@ # updater -> A helper tool for modifying CUE files to override arbitrary values. -> Useful for updating Timoni bundles. +> A helper tool for updating CUE files, especially [Timoni] bundle files. -The `updater` CLI provides an interface for overriding existing concrete values in a given CUE file. -Normally, concrete values in CUE files are immutable and thus not possible to override using the CUE CLI. -However, in some cases, it may be desirable to override an existing concrete value. -This is especially true in GitOps scenarios where a source of truth needs to be updated. +The `updater` CLI provides an interface for performing common deployment operations within Project Catalyst. +It has various subcommands centered around supporting GitOps operations by updating CUE files, especially [Timoni] bundle files. +It can be used in tandem with a CI/CD provider to easily update CUE configuration files within an existing repository. ## Usage -The `updater` CLI is most commonly used to update Timoni bundle image tags. -Assuming you have a `bundle.cue` file like this: +The `updater` CLI can be used in various workflows. +This section will provide high-level examples of the different workflows it can be used to support. + +### Updating CUE files + +By design, CUE files are intended to be immutable. +If a CUE file has a concrete value in a field, it's not possible to easily override it using the CUE CLI. +The `updater` CLI provides a subcommand for easily overriding concrete values within an existing CUE file. +This is especially important for supporting GitOps patterns. + +Assuming we have an existing CUE file: + +```cue +foo: { + bar: 1 +} +``` + +We can update the value of `bar` with: + +```terminal +$ updater update file -f ./file.cue "foo.bar" "2" # Pass --in-place to update the file in place +foo: { + bar: 2 +} +``` + +### Updating Timoni bundle files + +A common GitOps flow is to update specific values within a [Timoni bundle file](https://timoni.sh/bundle/). +The `updater` CLI provides a specific subcommand for updating the `values` specified in a bundle. +Given the following bundle file: ```cue bundle: { @@ -25,19 +53,130 @@ bundle: { } namespace: "default" values: { - server: image: tag: "ed2951cf049e779bba8d97413653bb06d4c28144" + server: image: tag: "v0.1.0" } } } } ``` -You can update the value of `server.image.tag` like so: +We can update the value of `server.image.tag` with: + +```terminal +$ updater update bundle -f ./bundle.cue -i instance "server.image.tag" "v0.2.0" +# ... + values: { + server: image: tag: "v0.2.0" + } +# ... +``` + +### Mass updating Timoni bundle files + +The primary use of the `updater` CLI is for performing a mass update of Timoni bundle files. +This approach is quite opinionated and requires some additional context. + +#### Deployment files + +Much like Catalyst CI, the `updater` CLI assumes a mono-repo setup for applications. +Where Catalyst CI expects each application to contain an `Earthfile` describing how to build the application, the `updater` CLI +expects each application to contain a `deployment.yml` file describing how to update the application's associated deployment. +For example: + +* `/app1` + * `Earthfile` + * `deployment.yml` +* `/app2` + * `Earthfile` + * `deployment.yml` + +Each deployment file contains a set of override operations that instruct the `updater` CLI on which Timoni bundle files to update. +For example: + +```yaml +overrides: + - app: app1 + instance: app1 # Can be omitted if same as app + path: server.image.tag + value: v0.2.0 +``` + +The exact purpose of each of these fields will become more clear later. + +#### Deployment repository + +The `updater` CLI assumes a mono-repo deployment repository exists containing Timoni bundle files for each environment. +An example structure would look like this: + +* `/bundles` + * `/dev` + * `/app1` + * `bundle.cue` + * `/app2` + * `bundle.cue` + * `/staging` + * `/app1` + * `bundle.cue` + +The root directory (`/bundles`) has subdirectories for each environment and each environment has subdirectories for every +application. +Each application in turn has a dedicated `bundle.cue` that contains the deployment code for the application. + +#### Updating files + +Given the previous example structure and deployment file, the `updater` CLI can automatically update the correct bundle file. +First, we will use the `scan` subcommand to automatically discover all `deployment.yml` files and parse their respective overrides: + +```terminal +$ updater scan . +[{"app":"app1","instance":"app1","path":"server.image.tag","value":"v0.2.0"}] +``` + +This produces a JSON output containing a list of overrides. +The JSON output can be used stand-alone, however, it can also be passed directly to the `update deployments` subcommand: + +```terminal +$ updater scan . | updater update deployments -e dev /path/to/deployment-repo/bundles +# Empty output +``` + +The above command performs the following for each given override: + +* Constructs a path to the bundle file: `///bundle.cue` +* Constructs a path to override: `bundle.instances..values.` +* Overrides the constructed value path within the constructed file path with the override value + +Using the previous examples, it would perform the following: + +* Constructs a path to the bundle file: `/path/to/deployment-repo/bundles/dev/app1/bundle.cue` +* Constructs a path to override: `bundle.instances.app1.values.server.image.tag` +* Overrides the constructed value path with `v0.2.0` + +This setup allows updating arbitrary bundle files and their respective values by defining a single `deployment.yml` file at the root +of a given application. +In a normal GitOps flow, the changes would then be committed to the deployment repository and picked up by a GitOps operator. + +#### Templating + +The previous example hardcoded an override value in the `deployment.yml`. +In some cases, the value is only known at runtime (i.e., when the CI/CD system is running). +For these cases, it's possible to override arbitrary "template" literals: + +```yaml +overrides: + - app: app1 + path: server.image.tag + value: GIT_SHA +``` + +When the CI/CD is performing an update, it can pass the value for this template literal like so (assuming GitHub Actions): -```shell -updater -b bundle.cue "bundle.instances.instance.values.server.image.tag" "0fe74bf77739a3ef78de5fcc81c5c7a8dcae6199" +```terminal +$ updater scan -t "GIT_SHA=${{ github.sha }}" . | updater update deployments -e dev /path/to/deployment-repo/bundles +# Empty output ``` -The `updater` CLI will overwrite the image tag with the provided one and update the `bundle.cue` file in place. -Note that the CLI uses the CUE API underneath the hood which may format the existing CUE syntax slightly differently. -In some cases, the resulting syntax might be a bit unsightly, so it's recommended to run `cue fmt` on the file after processing. +Prior to updating the bundle files, the `updater` CLI will replace all occurrences of `GIT_SHA` with the current commit SHA. +This allows dynamically updating the image tag of the application deployment at runtime if you tag images using the commit SHA + +[timoni]: https://timoni.sh/ diff --git a/tools/updater/cmd/main.go b/tools/updater/cmd/main.go index 6f2c8f1ba..b3d935037 100644 --- a/tools/updater/cmd/main.go +++ b/tools/updater/cmd/main.go @@ -1,48 +1,172 @@ package main -// cspell: words afero alecthomas cuelang cuecontext cuectx existingfile mheers Timoni nolint +// cspell: words afero alecthomas cuectx existingdir existingfile Timoni nolint import ( + "encoding/json" + "fmt" + "io" "os" + "path" + "strings" - "cuelang.org/go/cue" - "cuelang.org/go/cue/cuecontext" - "cuelang.org/go/cue/format" "github.com/alecthomas/kong" "github.com/input-output-hk/catalyst-ci/tools/updater/pkg" - ch "github.com/mheers/cue-helper/pkg/value" "github.com/spf13/afero" ) var cli struct { - BundleFile string `type:"existingfile" short:"b" help:"Path to the Timoni bundle file to modify." required:"true"` - Path string `arg:"" help:"A dot separated path to the value to override (must already exist)."` - Value string `arg:"" help:"The value to override the value at the path with."` + Scan scanCmd `cmd:"" help:"Scans a directory for deployment files."` + Update updateCmd `cmd:"" help:"Overrides a target path in a CUE file with the given value."` } -func main() { - ctx := kong.Parse(&cli, - kong.Name("updater"), - kong.Description("A helper tool for modifying CUE files to override arbitrary values. Useful for updating Timoni bundles.")) +type updateCmd struct { + Bundle updateBundleCmd `cmd:"" help:"Overrides a target path in a Timoni bundle values field with the given value."` + Deployments updateDeploymentsCmd `cmd:"" help:"Performs a mass update on Timoni bundle files using given input data."` + File updateFileCmd `cmd:"" help:"Overrides a target path in a CUE file with the given value."` +} - cuectx := cuecontext.New() - v, err := pkg.ReadFile(cuectx, cli.BundleFile, afero.NewOsFs()) - ctx.FatalIfErrorf(err) +type scanCmd struct { + Path string `arg:"" help:"The path to scan for deployment files." type:"existingdir" required:"true"` + Template []string `short:"t" help:"A key/value pair used to override constant values in deployment configurations."` +} - if !v.LookupPath(cue.ParsePath(cli.Path)).Exists() { - ctx.Fatalf("path %q does not exist", cli.Path) +func (c *scanCmd) Run() error { + files, err := pkg.ScanForDeploymentFiles(c.Path, afero.NewOsFs()) + if err != nil { + return err } - v, err = ch.Replace(v, cli.Path, cli.Value) - ctx.FatalIfErrorf(err) + overrides := []pkg.OverrideConfig{} + for _, file := range files { + overrides = append(overrides, file.Overrides...) + } - node := v.Syntax(cue.Final(), cue.Concrete(true), cue.Docs(true)) - src, err := format.Node(node) - ctx.FatalIfErrorf(err) + for _, template := range c.Template { + pair := strings.Split(template, "=") + pkg.ApplyTemplateValue(overrides, pair[0], pair[1]) + } + + output, err := json.Marshal(overrides) + if err != nil { + return fmt.Errorf("failed to marshal overrides: %v", err) + } - if err := os.WriteFile(cli.BundleFile, src, 0644); err != nil { //nolint:gosec - ctx.Fatalf("failed to write file %q: %v", cli.BundleFile, err) + fmt.Print(string(output)) + + return nil +} + +type updateBundleCmd struct { + BundleFile string `type:"existingfile" short:"f" help:"Path to the bundle file to update." required:"true"` + Instance string `short:"i" help:"The instance to update." required:"true"` + InPlace bool `help:"Update the file in place."` + Path string `arg:"" help:"A dot separated path to the value to update (must already exist)."` + Value string `arg:"" help:"The value to updates the value at the path with."` +} + +func (c *updateBundleCmd) Run() error { + path := fmt.Sprintf("bundle.instances.%s.values.%s", c.Instance, c.Path) + + src, err := pkg.UpdateFile(c.BundleFile, path, c.Value, afero.NewOsFs()) + if err != nil { + return err } + if c.InPlace { + if err := os.WriteFile(c.BundleFile, src, 0644); err != nil { //nolint:gosec + return fmt.Errorf("failed to write file %q: %v", c.BundleFile, err) + } + } else { + fmt.Print(string(src)) + } + + return nil +} + +type updateDeploymentsCmd struct { + RootDir string `arg:"" type:"existingdir" help:"The root directory where Timoni bundle files are located." required:"true"` + Environment string `short:"e" help:"The environment to update." required:"true"` + Input string `short:"i" help:"The path to the input data file (can be passed via stdin)."` +} + +func (c *updateDeploymentsCmd) Run() error { + var data []byte + + _, err := os.Stat(c.Input) + if os.IsNotExist(err) { + data, err = io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("failed to read input data: %v", err) + } + } else { + data, err = os.ReadFile(c.Input) + if err != nil { + return fmt.Errorf("failed to read input data from %s: %v", c.Input, err) + } + } + + overrides := []pkg.OverrideConfig{} + if err := json.Unmarshal(data, &overrides); err != nil { + return fmt.Errorf("failed to parse input data: %v", err) + } + + for _, override := range overrides { + bundlePath := path.Join(c.RootDir, c.Environment, override.App, "bundle.cue") + if _, err := os.Stat(bundlePath); os.IsNotExist(err) { + return fmt.Errorf("bundle file %q does not exist", bundlePath) + } + + var path string + if override.Instance == "" { + path = fmt.Sprintf("bundle.instances.%s.values.%s", override.App, override.Path) + } else { + path = fmt.Sprintf("bundle.instances.%s.values.%s", override.Instance, override.Path) + } + + src, err := pkg.UpdateFile(bundlePath, path, override.Value, afero.NewOsFs()) + if err != nil { + return fmt.Errorf("failed to update bundle file %q: %v", bundlePath, err) + } + + if err := os.WriteFile(bundlePath, src, 0644); err != nil { //nolint:gosec + return fmt.Errorf("failed to write bundle file %q: %v", bundlePath, err) + } + } + + return nil +} + +type updateFileCmd struct { + File string `type:"existingfile" short:"f" help:"Path to the CUE file to update." required:"true"` + InPlace bool `help:"Update the file in place."` + Path string `arg:"" help:"A dot separated path to the value to update (must already exist)."` + Value string `arg:"" help:"The value to updates the value at the path with."` +} + +func (c *updateFileCmd) Run() error { + src, err := pkg.UpdateFile(c.File, c.Path, c.Value, afero.NewOsFs()) + if err != nil { + return err + } + + if c.InPlace { + if err := os.WriteFile(c.File, src, 0644); err != nil { //nolint:gosec + return fmt.Errorf("failed to write file %q: %v", c.File, err) + } + } else { + fmt.Print(string(src)) + } + + return nil +} + +func main() { + ctx := kong.Parse(&cli, + kong.Name("updater"), + kong.Description("A helper tool for modifying CUE files to override arbitrary values. Useful for updating Timoni bundles.")) + + err := ctx.Run() + ctx.FatalIfErrorf(err) os.Exit(0) } diff --git a/tools/updater/go.mod b/tools/updater/go.mod index 9293266cf..05ceb603a 100644 --- a/tools/updater/go.mod +++ b/tools/updater/go.mod @@ -1,6 +1,8 @@ module github.com/input-output-hk/catalyst-ci/tools/updater -go 1.20 +go 1.21 + +toolchain go1.21.5 require ( cuelang.org/go v0.7.0 @@ -9,6 +11,7 @@ require ( github.com/onsi/ginkgo/v2 v2.9.2 github.com/onsi/gomega v1.27.5 github.com/spf13/afero v1.11.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -26,5 +29,4 @@ require ( golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.16.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/tools/updater/go.sum b/tools/updater/go.sum index 66e80d5f8..adfac52fb 100644 --- a/tools/updater/go.sum +++ b/tools/updater/go.sum @@ -1,10 +1,13 @@ cuelabs.dev/go/oci/ociregistry v0.0.0-20231103182354-93e78c079a13 h1:zkiIe8AxZ/kDjqQN+mDKc5BxoVJOqioSdqApjc+eB1I= +cuelabs.dev/go/oci/ociregistry v0.0.0-20231103182354-93e78c079a13/go.mod h1:XGKYSMtsJWfqQYPwq51ZygxAPqpEUj/9bdg16iDPTAA= cuelang.org/go v0.7.0 h1:gMztinxuKfJwMIxtboFsNc6s8AxwJGgsJV+3CuLffHI= cuelang.org/go v0.7.0/go.mod h1:ix+3dM/bSpdG9xg6qpCgnJnpeLtciZu+O/rDbywoMII= github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0= +github.com/alecthomas/assert/v2 v2.1.0/go.mod h1:b/+1DI2Q6NckYi+3mXyH3wFb8qG37K/DuK80n7WefXA= github.com/alecthomas/kong v0.8.1 h1:acZdn3m4lLRobeh3Zi2S2EpnXTd1mOL6U7xVml+vfkY= github.com/alecthomas/kong v0.8.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE= +github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -14,14 +17,17 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/proto v1.10.0 h1:pDGyFRVV5RvV+nkBK9iy3q67FBy9Xa7vwrOTE+g5aGw= +github.com/emicklei/proto v1.10.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= @@ -29,15 +35,20 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubevela/workflow v0.6.0 h1:fYXviOYD5zqHs3J61tNbM4HZ85EcZlPm7Fyz8Q5o9Fk= github.com/kubevela/workflow v0.6.0/go.mod h1:sjLcYqKHKeCQ+w77gijoNILwIShJKnCU+e3q7ETtZGI= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mheers/cue-helper v0.0.0-20231214081257-a325036f9a0d h1:JZsFf1WWJjBq7swih1HeWbmnh5uKZ2Ol2HSZ8o6kcK8= github.com/mheers/cue-helper v0.0.0-20231214081257-a325036f9a0d/go.mod h1:TvPPtiz/noXLe+NDP3XJLg8Y0ROt3m6rVlqUYlLuba0= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9ZPiLVHXz3UFw2+psEX+gYcto= github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuHZgRjZUWWuH1DTxCtxbHDOIJsudS8jzY= github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= @@ -45,19 +56,25 @@ github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxe github.com/onsi/gomega v1.27.5 h1:T/X6I0RNFw/kTqgfkZPcQ5KU6vCnWNBGdtrIx2dpGeQ= github.com/onsi/gomega v1.27.5/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= +github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0 h1:sadMIsgmHpEOGbUs6VtHBXRR1OHevnj7hLx9ZcdNGW4= +github.com/protocolbuffers/txtpbfmt v0.0.0-20230328191034-3462fbc510c0/go.mod h1:jgxiZysxFPM+iWKwQwPR+y+Jvo54ARd4EisXxKYpB5c= github.com/rogpeppe/go-internal v1.11.1-0.20231026093722-fa6a31e0812c h1:fPpdjePK1atuOg28PXfNSqgwf9I/qD1Hlo39JFwKBXk= +github.com/rogpeppe/go-internal v1.11.1-0.20231026093722-fa6a31e0812c/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -68,8 +85,10 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/updater/pkg/cue.go b/tools/updater/pkg/cue.go index 29d2eaf3f..ce6eb8ac5 100644 --- a/tools/updater/pkg/cue.go +++ b/tools/updater/pkg/cue.go @@ -1,17 +1,20 @@ package pkg -// cspell: words afero cuelang +// cspell: words afero cuecontext cuelang mheers import ( "fmt" "cuelang.org/go/cue" + "cuelang.org/go/cue/cuecontext" + "cuelang.org/go/cue/format" + ch "github.com/mheers/cue-helper/pkg/value" "github.com/spf13/afero" ) // ReadFile reads a CUE file and returns a cue.Value. -func ReadFile(ctx *cue.Context, path string, os afero.Fs) (cue.Value, error) { - contents, err := afero.ReadFile(os, path) +func ReadFile(ctx *cue.Context, path string, fs afero.Fs) (cue.Value, error) { + contents, err := afero.ReadFile(fs, path) if err != nil { return cue.Value{}, fmt.Errorf("failed to read file %q: %w", path, err) } @@ -23,3 +26,28 @@ func ReadFile(ctx *cue.Context, path string, os afero.Fs) (cue.Value, error) { return v, nil } + +// UpdateFile updates a CUE file at the given path with the given value and returns the updated file contents. +func UpdateFile(filePath, path string, value interface{}, fs afero.Fs) ([]byte, error) { + v, err := ReadFile(cuecontext.New(), filePath, fs) + if err != nil { + return nil, err + } + + if !v.LookupPath(cue.ParsePath(path)).Exists() { + return nil, fmt.Errorf("path %q does not exist", path) + } + + v, err = ch.Replace(v, path, value) + if err != nil { + return nil, fmt.Errorf("failed to replace value at path %q: %w", path, err) + } + + node := v.Syntax(cue.Final(), cue.Concrete(true), cue.Docs(true)) + src, err := format.Node(node) + if err != nil { + return nil, fmt.Errorf("failed to format CUE file: %w", err) + } + + return src, nil +} diff --git a/tools/updater/pkg/cue_test.go b/tools/updater/pkg/cue_test.go index 738596f00..3dc8c615e 100644 --- a/tools/updater/pkg/cue_test.go +++ b/tools/updater/pkg/cue_test.go @@ -3,6 +3,7 @@ package pkg_test // cspell: words afero cuelang cuecontext import ( + "cuelang.org/go/cue" "cuelang.org/go/cue/cuecontext" "github.com/input-output-hk/catalyst-ci/tools/updater/pkg" . "github.com/onsi/ginkgo/v2" @@ -12,19 +13,19 @@ import ( var _ = Describe("Cue", func() { Describe("ReadFile", func() { - var os afero.Fs + var fs afero.Fs BeforeEach(func() { - os = afero.NewMemMapFs() + fs = afero.NewMemMapFs() }) When("the file exists", func() { It("returns a cue.Value", func() { - err := afero.WriteFile(os, "foo.cue", []byte("foo: 1"), 0644) + err := afero.WriteFile(fs, "foo.cue", []byte("foo: 1"), 0644) Expect(err).ToNot(HaveOccurred()) ctx := cuecontext.New() - v, err := pkg.ReadFile(ctx, "foo.cue", os) + v, err := pkg.ReadFile(ctx, "foo.cue", fs) Expect(err).ToNot(HaveOccurred()) Expect(v).ToNot(BeNil()) }) @@ -34,9 +35,51 @@ var _ = Describe("Cue", func() { It("returns an error", func() { ctx := cuecontext.New() - _, err := pkg.ReadFile(ctx, "foo.cue", os) + _, err := pkg.ReadFile(ctx, "foo.cue", fs) Expect(err).To(HaveOccurred()) }) }) }) + + Describe("UpdateFile", func() { + var fs afero.Fs + + When("updating a file", func() { + BeforeEach(func() { + fs = afero.NewMemMapFs() + }) + + When("the file exists", func() { + BeforeEach(func() { + err := afero.WriteFile(fs, "foo.cue", []byte("foo: 1"), 0644) + Expect(err).ToNot(HaveOccurred()) + }) + + When("the path exists", func() { + It("updates the file", func() { + src, err := pkg.UpdateFile("foo.cue", "foo", 2, fs) + Expect(err).ToNot(HaveOccurred()) + + ctx := cuecontext.New() + v := ctx.CompileBytes(src) + Expect(v.LookupPath(cue.ParsePath("foo")).Int64()).To(Equal(int64(2))) + }) + }) + + When("the path does not exist", func() { + It("returns an error", func() { + _, err := pkg.UpdateFile("foo.cue", "bar", 2, fs) + Expect(err).To(HaveOccurred()) + }) + }) + }) + + When("the file does not exist", func() { + It("returns an error", func() { + _, err := pkg.UpdateFile("foo.cue", "foo", 2, fs) + Expect(err).To(HaveOccurred()) + }) + }) + }) + }) }) diff --git a/tools/updater/pkg/deployment.go b/tools/updater/pkg/deployment.go new file mode 100644 index 000000000..8f1719a0a --- /dev/null +++ b/tools/updater/pkg/deployment.go @@ -0,0 +1,75 @@ +package pkg + +// cspell: words afero + +import ( + "fmt" + "os" + + "github.com/spf13/afero" + "gopkg.in/yaml.v3" +) + +// Filename is the static name of a deployment configuration file. +const Filename = "deployment.yml" + +// DeploymentFile represents a deployment configuration file. +type DeploymentFile struct { + Overrides []OverrideConfig `json:"overrides" yaml:"overrides"` +} + +// OverrideConfig represents configuration for overriding a value in a CUE file. +type OverrideConfig struct { + App string `json:"app" yaml:"app"` + Instance string `json:"instance" yaml:"instance"` + Path string `json:"path" yaml:"path"` + Value string `json:"value" yaml:"value"` +} + +// ScanForDeploymentFiles scans a directory for deployment configuration files. +func ScanForDeploymentFiles(dir string, fs afero.Fs) ([]DeploymentFile, error) { + + files := []DeploymentFile{} + err := afero.Walk(fs, dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + if info.Name() == Filename { + contents, err := afero.ReadFile(fs, path) + if err != nil { + return fmt.Errorf("failed to read deployment file at %q: %v", path, err) + } + + deploymentFile := DeploymentFile{} + if err := yaml.Unmarshal(contents, &deploymentFile); err != nil { + // If we can't unmarshal the file, we assume it's not a deployment file and just log a warning. + fmt.Fprintf(os.Stderr, "warning: failed to parse deployment file %q: %v", path, err) + return nil + } + + files = append(files, deploymentFile) + } + + return nil + }) + + if err != nil { + return nil, err + } + + return files, nil +} + +// ApplyTemplateValue applies a template value to a list of overrides. +func ApplyTemplateValue(overrides []OverrideConfig, key, value string) { + for i, override := range overrides { + if override.Value == key { + overrides[i].Value = value + } + } +} diff --git a/tools/updater/pkg/deployment_test.go b/tools/updater/pkg/deployment_test.go new file mode 100644 index 000000000..761152937 --- /dev/null +++ b/tools/updater/pkg/deployment_test.go @@ -0,0 +1,121 @@ +package pkg_test + +// cspell: words afero + +import ( + "fmt" + + "github.com/input-output-hk/catalyst-ci/tools/updater/pkg" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/spf13/afero" + "gopkg.in/yaml.v3" +) + +var _ = Describe("Deployment", func() { + Describe("ScanForDeploymentFiles", func() { + var fs afero.Fs + + When("scanning a directory with valid deployment files", func() { + BeforeEach(func() { + fs = afero.NewMemMapFs() + + Expect(fs.MkdirAll("files", 0755)).To(Succeed()) + for i := range []int{1, 2, 3} { + DeploymentFile := pkg.DeploymentFile{ + Overrides: []pkg.OverrideConfig{ + { + App: "foo", + Path: "bar", + Value: "baz", + }, + }, + } + + contents, err := yaml.Marshal(DeploymentFile) + Expect(err).NotTo(HaveOccurred()) + + Expect(fs.MkdirAll(fmt.Sprintf("files/%d", i), 0755)).To(Succeed()) + Expect(afero.WriteFile(fs, fmt.Sprintf("files/%d/deployment.yml", i), contents, 0644)).To(Succeed()) + } + }) + + It("should return a list of all deployment files", func() { + files, err := pkg.ScanForDeploymentFiles("files", fs) + Expect(err).NotTo(HaveOccurred()) + Expect(files).To(HaveLen(3)) + }) + + It("should correctly parse the deployment files", func() { + files, err := pkg.ScanForDeploymentFiles("files", fs) + Expect(err).NotTo(HaveOccurred()) + + for _, file := range files { + Expect(file.Overrides).To(HaveLen(1)) + Expect(file.Overrides[0].App).To(Equal("foo")) + Expect(file.Overrides[0].Path).To(Equal("bar")) + Expect(file.Overrides[0].Value).To(Equal("baz")) + } + }) + }) + + When("scanning a directory with an invalid deployment file", func() { + BeforeEach(func() { + fs = afero.NewMemMapFs() + + Expect(fs.MkdirAll("files", 0755)).To(Succeed()) + for i := range []int{1, 2} { + DeploymentFile := pkg.DeploymentFile{ + Overrides: []pkg.OverrideConfig{ + { + App: "foo", + Path: "bar", + Value: "baz", + }, + }, + } + + contents, err := yaml.Marshal(DeploymentFile) + Expect(err).NotTo(HaveOccurred()) + + Expect(fs.MkdirAll(fmt.Sprintf("files/%d", i), 0755)).To(Succeed()) + Expect(afero.WriteFile(fs, fmt.Sprintf("files/%d/deployment.yml", i), contents, 0644)).To(Succeed()) + } + + Expect(afero.WriteFile(fs, "files/deployment.yml", []byte("invalid"), 0644)).To(Succeed()) + }) + + It("should ignore the invalid file", func() { + files, err := pkg.ScanForDeploymentFiles("files", fs) + Expect(err).NotTo(HaveOccurred()) + Expect(files).To(HaveLen(2)) + }) + }) + }) + + Describe("ApplyTemplateValue", func() { + var overrides []pkg.OverrideConfig + When("applying a template value to a list of overrides", func() { + BeforeEach(func() { + overrides = []pkg.OverrideConfig{ + { + App: "foo", + Path: "bar", + Value: "baz", + }, + { + App: "foo", + Path: "bar", + Value: "qux", + }, + } + }) + + It("should update the value of the override with the given key", func() { + pkg.ApplyTemplateValue(overrides, "baz", "qux") + Expect(overrides[0].Value).To(Equal("qux")) + Expect(overrides[1].Value).To(Equal("qux")) + }) + }) + }) +})