Embedded JavaScript pipeline PoC

clicommand/pipeline_eval.go

@@ -0,0 +1,149 @@
+package clicommand
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+
+ "github.com/buildkite/agent/v3/cliconfig"
+ "github.com/buildkite/agent/v3/js"
+ "github.com/buildkite/agent/v3/stdin"
+ "github.com/urfave/cli"
+)
+
+const evalDescription = `Usage:
+ buildkite-agent pipeline eval [options]
+
+Description:
+ Something something JavaScript?
+
+Example:
+ $ buildkite-agent pipeline eval buildkite.js
+
+ Evaluates buildkite.js as JavaScript and perhaps uploads the stdout as JSON/YAML pipeline?
+`
+
+type PipelineEvalConfig struct {
+ FilePath string `cli:"arg:0" label:"upload paths"`
+
+ // Global flags
+ Debug bool `cli:"debug"`
+ LogLevel string `cli:"log-level"`
+ NoColor bool `cli:"no-color"`
+ Experiments []string `cli:"experiment" normalize:"list"`
+ Profile string `cli:"profile"`
+}
+
+var PipelineEvalCommand = cli.Command{
+ Name: "eval",
+ Usage: "Evaluates a JavaScript pipeline",
+ Description: evalDescription,
+ Flags: []cli.Flag{
+ // Global flags
+ NoColorFlag,
+ DebugFlag,
+ LogLevelFlag,
+ ExperimentsFlag,
+ ProfileFlag,
+ },
+ Action: func(c *cli.Context) error {
+ // The configuration will be loaded into this struct
+ cfg := PipelineEvalConfig{}
+
+ loader := cliconfig.Loader{CLI: c, Config: &cfg}
+ warnings, err := loader.Load()
+ if err != nil {
+ fmt.Printf("%s", err)
+ os.Exit(1)
+ }
+
+ l := CreateLogger(&cfg)
+
+ // Now that we have a logger, log out the warnings that loading config generated
+ for _, warning := range warnings {
+ l.Warn("%s", warning)
+ }
+
+ // Setup any global configuration options
+ done := HandleGlobalFlags(l, cfg)
+ defer done()
+
+ // Find the pipeline file either from STDIN or the first
+ // argument
+ var input []byte
+ var filename string
+
+ if cfg.FilePath != "" {
+ l.Info("Reading pipeline config from \"%s\"", cfg.FilePath)
+
+ filename = filepath.Base(cfg.FilePath)
+ input, err = os.ReadFile(cfg.FilePath)
+ if err != nil {
+ l.Fatal("Failed to read file: %s", err)
+ }
+ } else if stdin.IsReadable() {
+ l.Info("Reading pipeline config from STDIN")
+
+ // Actually read the file from STDIN
+ filename = "(stdin)"
+ input, err = io.ReadAll(os.Stdin)
+ if err != nil {
+ l.Fatal("Failed to read from STDIN: %s", err)
+ }
+ } else {
+ l.Info("Searching for pipeline config...")
+
+ paths := []string{
+ "buildkite.js",
+ filepath.FromSlash(".buildkite/buildkite.js"),
+ filepath.FromSlash("buildkite/buildkite.js"),
+ }
+
+ // Collect all the files that exist
+ exists := []string{}
+ for _, path := range paths {
+ if _, err := os.Stat(path); err == nil {
+ exists = append(exists, path)
+ }
+ }
+
+ // If more than 1 of the config files exist, throw an
+ // error. There can only be one!!
+ if len(exists) > 1 {
+ l.Fatal("Found multiple configuration files: %s. Please only have 1 configuration file present.", strings.Join(exists, ", "))
+ } else if len(exists) == 0 {
+ l.Fatal("Could not find a default pipeline configuration file. See `buildkite-agent pipeline upload --help` for more information.")
+ }
+
+ found := exists[0]
+
+ l.Info("Found config file \"%s\"", found)
+
+ // Read the default file
+ filename = path.Base(found)
+ input, err = os.ReadFile(found)
+ if err != nil {
+ l.Fatal("Failed to read file \"%s\" (%s)", found, err)
+ }
+ }
+
+ pipelineYAML, err := js.EvalJS(filename, input, l)
+ if err != nil {
+ return fmt.Errorf("JavaScript evaluation: %w", err)
+ }
+
+ n, err := c.App.Writer.Write(pipelineYAML)
+ if err != nil {
+ return nil
+ }
+ if n != len(pipelineYAML) {
+ return errors.New("short write")
+ }
+
+ return nil
+ },
+}

go.mod

@@ -58,9 +58,13 @@ require (
  github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect
  github.com/davecgh/go-spew v1.1.1 // indirect
  github.com/dgraph-io/ristretto v0.1.0 // indirect
+ github.com/dlclark/regexp2 v1.7.0 // indirect
+ github.com/dop251/goja v0.0.0-20221106173738-3b8a68ca89b4 // indirect
+ github.com/dop251/goja_nodejs v0.0.0-20221009164102-3aa5028e57f6 // indirect
  github.com/dustin/go-humanize v1.0.0 // indirect
  github.com/go-logr/logr v1.2.3 // indirect
  github.com/go-logr/stdr v1.2.2 // indirect
+ github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
  github.com/golang/glog v1.0.0 // indirect
  github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
  github.com/golang/protobuf v1.5.2 // indirect

go.sum

@@ -105,6 +105,7 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -116,6 +117,17 @@ github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/Lu
 github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo=
+github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
+github.com/dop251/goja v0.0.0-20220815083517-0c74f9139fd6/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs=
+github.com/dop251/goja v0.0.0-20221106173738-3b8a68ca89b4 h1:arM6Tq1Ba+a9FWuq3S6Qgrfd5MD0slQdMnCKI2VclFg=
+github.com/dop251/goja v0.0.0-20221106173738-3b8a68ca89b4/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs=
+github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
+github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
+github.com/dop251/goja_nodejs v0.0.0-20221009164102-3aa5028e57f6 h1:p3QZwRRfCN7Qr3GNBTMKBkLFjEm3DHR4MaJABvsiqgk=
+github.com/dop251/goja_nodejs v0.0.0-20221009164102-3aa5028e57f6/go.mod h1:+CJy9V5cGycP5qwp6RM5jLg+TFEMyGtD7A9xUbU/BOQ=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
@@ -140,6 +152,8 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
+github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
 github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -251,9 +265,13 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53 h1:tGfIHhDghvEnneeRhODvGYOt305TPwingKt6p90F4MU=
@@ -285,6 +303,7 @@ github.com/rjeczalik/interfaces v0.1.1 h1:xhFQNGtz3T3CQgtJJwWn+i3Ekl1WeObh7wtTtC
 github.com/rjeczalik/interfaces v0.1.1/go.mod h1:TNwD+kCGmXYrXksRDD5ikspp08m/Aosbr67zVLMjnOY=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sasha-s/go-deadlock v0.0.0-20180226215254-237a9547c8a5 h1:T7hUw7pBSINuHQyWwMdfIWZZH5M3ju4yXIbuV/Upp+4=
@@ -811,11 +830,14 @@ gopkg.in/DataDog/dd-trace-go.v1 v1.43.1/go.mod h1:YL9g+nlUY7ByCffD5pDytAqy99GNby
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=