From c0c4f7166b571ce2fe9ec42532f6190b8e34115f Mon Sep 17 00:00:00 2001 From: Jordan Winters Date: Mon, 18 Apr 2022 20:41:46 -0400 Subject: [PATCH 1/2] Add support for deploying a pre-rendered jobspec. Render separately with nomad/HCL2 but still make use of levant's deployment watcher. --- command/deploy.go | 87 +++++++-- docs/commands.md | 8 + helper/files.go | 63 +++++++ helper/files_test.go | 48 +++++ helper/test-fixtures/demojob.json | 298 ++++++++++++++++++++++++++++++ 5 files changed, 491 insertions(+), 13 deletions(-) create mode 100644 helper/test-fixtures/demojob.json diff --git a/command/deploy.go b/command/deploy.go index c3817d226..3067191c7 100644 --- a/command/deploy.go +++ b/command/deploy.go @@ -2,6 +2,7 @@ package command import ( "fmt" + "os" "strings" "github.com/hashicorp/levant/helper" @@ -33,6 +34,9 @@ Arguments: TEMPLATE nomad job template If no argument is given we look for a single *.nomad file + Note that the -pre-rendered flag changes the expectation for this arg + (See below). + General Options: @@ -84,6 +88,12 @@ General Options: Specify the format of Levant's logs. Valid values are HUMAN or JSON. The default is HUMAN. + -pre-rendered + When used, TEMPLATE arg must instead specify a .json file, which must + contain a pre-rendered jobspec. + Alternatively you can omit the TEMPLATE arg and pipe JSON to levant deploy. + Not compatible with -var, -var-file, -vault, -vault-token, -consul-address. + -var-file= Used in conjunction with the -job-file will deploy a templated job to your Nomad cluster. You can repeat this flag multiple times to supply multiple var-files. @@ -102,6 +112,7 @@ func (c *DeployCommand) Run(args []string) int { var err error var level, format string + var preRendered bool config := &levant.DeployConfig{ Client: &structs.ClientConfig{}, @@ -123,12 +134,14 @@ func (c *DeployCommand) Run(args []string) int { flags.BoolVar(&config.Plan.IgnoreNoChanges, "ignore-no-changes", false, "") flags.StringVar(&level, "log-level", "INFO", "") flags.StringVar(&format, "log-format", "HUMAN", "") + flags.BoolVar(&preRendered, "pre-rendered", false, "") flags.StringVar(&config.Deploy.VaultToken, "vault-token", "", "") flags.BoolVar(&config.Deploy.EnvVault, "vault", false, "") flags.Var((*helper.FlagStringSlice)(&config.Template.VariableFiles), "var-file", "") if err = flags.Parse(args); err != nil { + c.UI.Error("ERROR: CLI flag parsing error.") return 1 } @@ -145,24 +158,72 @@ func (c *DeployCommand) Run(args []string) int { return 1 } - if len(args) == 1 { - config.Template.TemplateFile = args[0] - } else if len(args) == 0 { - if config.Template.TemplateFile = helper.GetDefaultTmplFile(); config.Template.TemplateFile == "" { + if preRendered { + if config.Deploy.EnvVault || + config.Client.ConsulAddr != "" || + config.Deploy.VaultToken != "" || + len(c.Meta.flagVars) > 0 || + len(config.Template.VariableFiles) > 0 { + c.UI.Error(c.Help()) - c.UI.Error("\nERROR: Template arg missing and no default template found") + c.UI.Error("\nERROR: -pre-rendered cannot be used with -consul-addr, -var, -var-file, -vault, or -vault-token.") return 1 } + + isPipedInput, err := helper.IsPipedInput() + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + + if len(args) == 1 { + if isPipedInput { + c.UI.Error(c.Help()) + c.UI.Error("\nERROR: Cannot pass rendered JSON file and piped input at the same time.") + return 1 + } + jobFile := args[0] + config.Template.Job, err = helper.GetJobspecFromFile(jobFile) + + } else if len(args) == 0 { + if isPipedInput { + config.Template.Job, err = helper.GetJobspecFromIOReader(os.Stdin) + } else { + c.UI.Error(c.Help()) + c.UI.Error("\nERROR: No rendered JSON file specified and no piped input found.") + return 1 + } + + } else { + c.UI.Error(c.Help()) + return 1 + } + + if err != nil { + c.UI.Error(fmt.Sprintf("[ERROR] levant/command: %v", err)) + return 1 + } + } else { - c.UI.Error(c.Help()) - return 1 - } + if len(args) == 1 { + config.Template.TemplateFile = args[0] + } else if len(args) == 0 { + if config.Template.TemplateFile = helper.GetDefaultTmplFile(); config.Template.TemplateFile == "" { + c.UI.Error(c.Help()) + c.UI.Error("\nERROR: Template arg missing and no default template found") + return 1 + } + } else { + c.UI.Error(c.Help()) + return 1 + } - config.Template.Job, err = template.RenderJob(config.Template.TemplateFile, - config.Template.VariableFiles, config.Client.ConsulAddr, &c.Meta.flagVars) - if err != nil { - c.UI.Error(fmt.Sprintf("[ERROR] levant/command: %v", err)) - return 1 + config.Template.Job, err = template.RenderJob(config.Template.TemplateFile, + config.Template.VariableFiles, config.Client.ConsulAddr, &c.Meta.flagVars) + if err != nil { + c.UI.Error(fmt.Sprintf("[ERROR] levant/command: %v", err)) + return 1 + } } if config.Deploy.Canary > 0 { diff --git a/docs/commands.md b/docs/commands.md index b054200b1..c1da4fa7f 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -26,6 +26,8 @@ Levant supports a number of command line arguments which provide control over th * **-log-format** (string: "HUMAN") Specify the format of Levant's logs. Valid values are HUMAN or JSON +* **-pre-rendered** (bool: "false") Pass a pre-rendered jobspec to levant instead of a template file. TEMPLATE arg must instead refer to a JSON file containing valid jobspec JSON. Alternatively you can omit the TEMPLATE arg and pipe JSON directly to levant deploy. **Not compatible with -var, -var-file, -vault, -vault-token, -consul-address**. See full example below. + * **-var-file** (string: "") The variables file to render the template with. This flag can be specified multiple times to supply multiple variables files. * **-vault** (bool: false) This flag makes Levant load the Vault token from the current ENV. It can not be used at the same time as the `vault-token` flag. @@ -40,6 +42,12 @@ Full example: levant deploy -log-level=debug -address=nomad.devoops -var-file=var.yaml -var 'var=test' example.nomad ``` +Example of -pre-rendered, using nomad/HCL2 to render and levant to deploy: + +``` +nomad job run -output -var-file=vars/dev.hcl job.hcl | levant deploy -pre-rendered -log-level=DEBUG -force-count +``` + ### Dispatch: `dispatch` `dispatch` allows you to dispatch an instance of a Nomad parameterized job and utilise Levant's advanced job checking features to ensure the job reaches the correct running state. diff --git a/helper/files.go b/helper/files.go index b5b882d3b..2083dd010 100644 --- a/helper/files.go +++ b/helper/files.go @@ -1,12 +1,23 @@ package helper import ( + "bufio" + "encoding/json" + "fmt" + "io" + "io/ioutil" "os" "path/filepath" + nomad "github.com/hashicorp/nomad/api" "github.com/rs/zerolog/log" ) +// JobJSON is used to unmarshal pre-rendered/parsed jobspec JSON +type JobJSON struct { + Job nomad.Job `json:"Job"` +} + // GetDefaultTmplFile checks the current working directory for *.nomad files. // If only 1 is found we return the match. func GetDefaultTmplFile() (templateFile string) { @@ -42,3 +53,55 @@ func GetDefaultVarFile() (varFile string) { log.Debug().Msg("helper/files: no default var-file found") return "" } + +// GetJobspecFromBytes converts JSON passed as bytes to a nomad job the same +// as levant's RenderTemplate would return. +func GetJobspecFromBytes(src []byte) (job *nomad.Job, err error) { + var jobspec JobJSON + + err = json.Unmarshal(src, &jobspec) + if err != nil { + err = fmt.Errorf("helper/files: error parsing JSON: %w", err) + } + + return &jobspec.Job, err +} + +// GetJobspecFromFile converts a JSON file to a nomad job the same as levant's +// RenderTemplate would return. +func GetJobspecFromFile(jobFile string) (job *nomad.Job, err error) { + src, err := ioutil.ReadFile(jobFile) + if err != nil { + return nil, err + } + + return GetJobspecFromBytes(src) +} + +// IsPipedInput determines if there is piped input to stdin. +func IsPipedInput() (isPiped bool, err error) { + info, err := os.Stdin.Stat() + if err != nil { + return false, err + } + + return info.Mode()&os.ModeCharDevice == 0, err +} + +// getJobspecFromIOReader reads bytes from a Reader and returns a nomad Job. +// Intended for os.Stdin but can be tested with any io.Reader. +// JSON must be valid and conform to the nomad.api.Job struct. +func GetJobspecFromIOReader(r io.Reader) (job *nomad.Job, err error) { + var runes []rune + reader := bufio.NewReader(r) + + for { + input, _, err := reader.ReadRune() + if err != nil && err == io.EOF { + break + } + runes = append(runes, input) + } + + return GetJobspecFromBytes([]byte(string(runes))) +} diff --git a/helper/files_test.go b/helper/files_test.go index 959bc29a8..93d2fdac6 100644 --- a/helper/files_test.go +++ b/helper/files_test.go @@ -1,6 +1,7 @@ package helper import ( + "bytes" "io/ioutil" "os" "reflect" @@ -81,3 +82,50 @@ func TestHelper_GetDefaultVarFile(t *testing.T) { os.Remove(tc.VarFile) } } + +func TestHelper_GetJobspecFromFile(t *testing.T) { + cases := []struct { + JobFile string + JobName string + }{ + {"test-fixtures/demojob.json", "demojob"}, + } + + for i, tc := range cases { + job, err := GetJobspecFromFile(tc.JobFile) + if err != nil { + t.Fatalf("case %d failed: %v", i, err) + } + + if !reflect.DeepEqual(*job.Name, tc.JobName) { + t.Fatalf("got: %#v, expected %#v", *job.Name, tc.JobName) + } + + } +} + +func TestHelper_GetJobspecFromIOReader(t *testing.T) { + cases := []struct { + JobFile string + JobName string + }{ + {"test-fixtures/demojob.json", "demojob"}, + } + + for i, tc := range cases { + src, err := ioutil.ReadFile(tc.JobFile) + if err != nil { + t.Fatalf("case %d failed: %v", i, err) + } + + job, err := GetJobspecFromIOReader(bytes.NewBuffer(src)) + if err != nil { + t.Fatalf("case %d failed: %v", i, err) + } + + if !reflect.DeepEqual(*job.Name, tc.JobName) { + t.Fatalf("got: %#v, expected %#v", *job.Name, tc.JobName) + } + + } +} diff --git a/helper/test-fixtures/demojob.json b/helper/test-fixtures/demojob.json new file mode 100644 index 000000000..686a553e3 --- /dev/null +++ b/helper/test-fixtures/demojob.json @@ -0,0 +1,298 @@ +{ + "Job": { + "Region": "secondary", + "Namespace": "demo1", + "ID": "demojob", + "Name": "demojob", + "Type": "service", + "Priority": null, + "AllAtOnce": null, + "Datacenters": [ + "dc1" + ], + "Constraints": [ + { + "LTarget": "${meta.namespace}", + "RTarget": "demo1", + "Operand": "=" + } + ], + "Affinities": null, + "TaskGroups": [ + { + "Name": "demojob", + "Count": null, + "Constraints": [ + { + "LTarget": "${node.class}", + "RTarget": "demo1-app", + "Operand": "=" + } + ], + "Affinities": null, + "Tasks": [ + { + "Name": "redis", + "Driver": "docker", + "User": "", + "Lifecycle": null, + "Config": { + "image": "redis:alpine", + "ports": [ + "redis" + ] + }, + "Constraints": null, + "Affinities": null, + "Env": null, + "Services": null, + "Resources": null, + "RestartPolicy": null, + "Meta": null, + "KillTimeout": null, + "LogConfig": null, + "Artifacts": null, + "Vault": { + "Policies": [ + "sudo" + ], + "Namespace": "demo1", + "Env": true, + "ChangeMode": "noop", + "ChangeSignal": null + }, + "Templates": null, + "DispatchPayload": null, + "VolumeMounts": null, + "Leader": false, + "ShutdownDelay": 0, + "KillSignal": "", + "Kind": "", + "ScalingPolicies": null + }, + { + "Name": "demojob", + "Driver": "docker", + "User": "", + "Lifecycle": null, + "Config": { + "dns_servers": [ + "${attr.unique.network.ip-address}" + ], + "image": "demojob:123", + "ports": [ + "api" + ] + }, + "Constraints": null, + "Affinities": null, + "Env": null, + "Services": null, + "Resources": null, + "RestartPolicy": null, + "Meta": null, + "KillTimeout": null, + "LogConfig": null, + "Artifacts": null, + "Vault": { + "Policies": [ + "sudo" + ], + "Namespace": "demo1", + "Env": true, + "ChangeMode": "noop", + "ChangeSignal": null + }, + "Templates": [ + { + "SourcePath": null, + "DestPath": "local/config.env", + "EmbeddedTmpl": " {{ with secret \"kv/postgres\" }}\n DB_PASSWORD = {{ index .Data.data.password }}\n {{ end }}\n ENV_FOO = {{ key \"demojob/config/foo\" }}\nENV_BAR = \"bar\"\n\n", + "ChangeMode": "restart", + "ChangeSignal": null, + "Splay": 5000000000, + "Perms": "0644", + "LeftDelim": null, + "RightDelim": null, + "Envvars": true, + "VaultGrace": null + } + ], + "DispatchPayload": null, + "VolumeMounts": null, + "Leader": false, + "ShutdownDelay": 0, + "KillSignal": "", + "Kind": "", + "ScalingPolicies": null + } + ], + "Spreads": null, + "Volumes": null, + "RestartPolicy": null, + "ReschedulePolicy": null, + "EphemeralDisk": null, + "Update": null, + "Migrate": null, + "Networks": [ + { + "Mode": "", + "Device": "", + "CIDR": "", + "IP": "", + "DNS": null, + "ReservedPorts": null, + "DynamicPorts": [ + { + "Label": "api", + "Value": 0, + "To": 5000, + "HostNetwork": "" + }, + { + "Label": "redis", + "Value": 0, + "To": 6379, + "HostNetwork": "" + } + ], + "MBits": null + } + ], + "Meta": null, + "Services": [ + { + "Id": "", + "Name": "demojob", + "Tags": [ + "traefik.enable=true", + "traefik.http.routers.demojob-http.entrypoints=web", + "traefik.http.routers.demojob-http.rule=Host(`demojob.demo1.dc1.domain.name`)", + "traefik.http.routers.demojob-http.middlewares=demojob-https", + "traefik.http.middlewares.demojob-https.redirectscheme.scheme=https", + "traefik.http.routers.demojob.tls=true", + "traefik.http.routers.demojob.entrypoints=websecure", + "traefik.http.routers.demojob.rule=Host(`demojob.demo1.dc1.domain.name`)" + ], + "CanaryTags": null, + "EnableTagOverride": false, + "PortLabel": "api", + "AddressMode": "host", + "Checks": [ + { + "Id": "", + "Name": "", + "Type": "tcp", + "Command": "", + "Args": null, + "Path": "", + "Protocol": "", + "PortLabel": "api", + "Expose": false, + "AddressMode": "", + "Interval": 10000000000, + "Timeout": 5000000000, + "InitialStatus": "", + "TLSSkipVerify": false, + "Header": null, + "Method": "", + "CheckRestart": null, + "GRPCService": "", + "GRPCUseTLS": false, + "TaskName": "", + "SuccessBeforePassing": 0, + "FailuresBeforeCritical": 0, + "Body": "", + "OnUpdate": "" + } + ], + "CheckRestart": null, + "Connect": null, + "Meta": null, + "CanaryMeta": null, + "TaskName": "", + "OnUpdate": "" + } + ], + "ShutdownDelay": null, + "StopAfterClientDisconnect": null, + "Scaling": { + "Min": 1, + "Max": 10, + "Policy": {}, + "Enabled": true, + "Type": "horizontal", + "ID": "", + "Namespace": "", + "Target": null, + "CreateIndex": 0, + "ModifyIndex": 0 + }, + "Consul": null + } + ], + "Update": { + "Stagger": 30000000000, + "MaxParallel": 1, + "HealthCheck": "checks", + "MinHealthyTime": 20000000000, + "HealthyDeadline": 60000000000, + "ProgressDeadline": 120000000000, + "Canary": null, + "AutoRevert": false, + "AutoPromote": null + }, + "Multiregion": null, + "Spreads": [ + { + "Attribute": "${attr.platform.aws.placement.availability-zone}", + "Weight": 100, + "SpreadTarget": [ + { + "Value": "row-1-1", + "Percent": 25 + }, + { + "Value": "row-1-2", + "Percent": 25 + }, + { + "Value": "row-2-1", + "Percent": 25 + }, + { + "Value": "row-2-2", + "Percent": 25 + } + ] + } + ], + "Periodic": null, + "ParameterizedJob": null, + "Reschedule": null, + "Migrate": { + "MaxParallel": 1, + "HealthCheck": "checks", + "MinHealthyTime": 10000000000, + "HealthyDeadline": 120000000000 + }, + "Meta": null, + "ConsulToken": null, + "VaultToken": null, + "Stop": null, + "ParentID": null, + "Dispatched": false, + "Payload": null, + "ConsulNamespace": null, + "VaultNamespace": null, + "NomadTokenID": null, + "Status": null, + "StatusDescription": null, + "Stable": null, + "Version": null, + "SubmitTime": null, + "CreateIndex": null, + "ModifyIndex": null, + "JobModifyIndex": null + } +} From 6a082c634a0de6288a239119e7804d5267043d36 Mon Sep 17 00:00:00 2001 From: Kamil Sitnik Date: Fri, 14 Oct 2022 12:30:38 -0400 Subject: [PATCH 2/2] use deployment watcher if deploy id exists --- levant/deploy.go | 35 ++++++----- test/deploy_test.go | 15 +++++ .../fixtures/deploy_fail_with_no_update.nomad | 58 +++++++++++++++++++ 3 files changed, 90 insertions(+), 18 deletions(-) create mode 100644 test/fixtures/deploy_fail_with_no_update.nomad diff --git a/levant/deploy.go b/levant/deploy.go index 01b0c33eb..b59cd3331 100644 --- a/levant/deploy.go +++ b/levant/deploy.go @@ -155,16 +155,7 @@ func (l *levantDeployment) deploy() (success bool) { switch *l.config.Template.Job.Type { case nomad.JobTypeService: - - // If the service job doesn't have an update stanza, the job will not use - // Nomad deployments. - if l.config.Template.Job.Update == nil { - log.Info().Msg("levant/deploy: job is not configured with update stanza, consider adding to use deployments") - return l.jobStatusChecker(&eval.EvalID) - } - log.Info().Msgf("levant/deploy: beginning deployment watcher for job") - // Get the deploymentID from the evaluationID so that we can watch the // deployment for end status. depID, err := l.getDeploymentID(eval.EvalID) @@ -173,6 +164,12 @@ func (l *levantDeployment) deploy() (success bool) { return } + if depID == "" { + log.Info().Msgf("levant/deploy: no deploy ID found for evaluation %s", eval.EvalID) + return l.jobStatusChecker(&eval.EvalID) + } + + log.Info().Msgf("levant/deploy: watching deployment %s for job", depID) // Get the success of the deployment and return if we have success. if success = l.deploymentWatcher(depID); success { return @@ -184,15 +181,17 @@ func (l *levantDeployment) deploy() (success bool) { return } - // If the job is not a canary job, then run the auto-revert checker, the - // current checking mechanism is slightly hacky and should be updated. - // The reason for this is currently the config.Job is populate from the - // rendered job and so a user could potentially not set canary meaning - // the field shows a null. - if l.config.Template.Job.Update.Canary == nil { - l.checkAutoRevert(dep) - } else if *l.config.Template.Job.Update.Canary == 0 { - l.checkAutoRevert(dep) + if l.config.Template.Job.Update != nil { + // If the job is not a canary job, then run the auto-revert checker, the + // current checking mechanism is slightly hacky and should be updated. + // The reason for this is currently the config.Job is populates from the + // rendered job and so a user could potentially not set canary meaning + // the field shows a null. + if l.config.Template.Job.Update.Canary == nil { + l.checkAutoRevert(dep) + } else if *l.config.Template.Job.Update.Canary == 0 { + l.checkAutoRevert(dep) + } } case nomad.JobTypeBatch: diff --git a/test/deploy_test.go b/test/deploy_test.go index f30e0e499..e2f031f4c 100644 --- a/test/deploy_test.go +++ b/test/deploy_test.go @@ -135,6 +135,21 @@ func TestDeploy_canary(t *testing.T) { }) } +func TestDeploy_failed_deploy_with_no_update(t *testing.T) { + acctest.Test(t, acctest.TestCase{ + Steps: []acctest.TestStep{ + { + Runner: acctest.DeployTestStepRunner{ + FixtureName: "deploy_fail_with_no_update.nomad", + }, + ExpectErr: true, + Check: acctest.CheckDeploymentStatus("failed"), + }, + }, + CleanupFunc: acctest.CleanupPurgeJob, + }) +} + func TestDeploy_lifecycle(t *testing.T) { acctest.Test(t, acctest.TestCase{ Steps: []acctest.TestStep{ diff --git a/test/fixtures/deploy_fail_with_no_update.nomad b/test/fixtures/deploy_fail_with_no_update.nomad new file mode 100644 index 000000000..1648dcb24 --- /dev/null +++ b/test/fixtures/deploy_fail_with_no_update.nomad @@ -0,0 +1,58 @@ +job "[[.job_name]]" { + datacenters = ["dc1"] + type = "service" + + group "test" { + count = 1 + + restart { + attempts = 1 + interval = "5s" + delay = "1s" + mode = "fail" + } + + ephemeral_disk { + size = 300 + } + + update { + max_parallel = 1 + min_healthy_time = "10s" + healthy_deadline = "1m" + } + + network { + port "http" { + to = 80 + } + } + + service { + name = "fake-service" + port = "http" + + check { + name = "alive" + type = "tcp" + interval = "10s" + timeout = "2s" + } + } + + task "alpine" { + driver = "docker" + config { + image = "alpine" + command = "sleep 1 && exit 1" + } + resources { + cpu = 100 + memory = 20 + network { + mbits = 10 + } + } + } + } +} \ No newline at end of file