Skip to content

Commit

Permalink
Embedded JavaScript pipeline PoC
Browse files Browse the repository at this point in the history
  • Loading branch information
pda committed Nov 14, 2022
1 parent 35ce452 commit 063c4e4
Show file tree
Hide file tree
Showing 7 changed files with 278 additions and 0 deletions.
217 changes: 217 additions & 0 deletions clicommand/pipeline_eval.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
package clicommand

import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"strings"

"github.com/buildkite/agent/v3/cliconfig"
"github.com/buildkite/agent/v3/resources"
"github.com/buildkite/agent/v3/stdin"
"github.com/buildkite/yaml"
"github.com/dop251/goja"
"github.com/dop251/goja_nodejs/console"
"github.com/dop251/goja_nodejs/process"
"github.com/dop251/goja_nodejs/require"
"github.com/urfave/cli"
)

const evalDescription = `Usage:
buildkite-agent pipeline eval [options]
Description:
Something something JavaScript?
Example:
$ buildkite-agent pipeline eval buildkite.js
Evaluates buildkite.js as JavaScript and perhaps uploads the stdout as JSON/YAML pipeline?
`

type PipelineEvalConfig struct {
FilePath string `cli:"arg:0" label:"upload paths"`

// Global flags
Debug bool `cli:"debug"`
LogLevel string `cli:"log-level"`
NoColor bool `cli:"no-color"`
Experiments []string `cli:"experiment" normalize:"list"`
Profile string `cli:"profile"`
}

var PipelineEvalCommand = cli.Command{
Name: "eval",
Usage: "Evaluates a JavaScript pipeline",
Description: evalDescription,
Flags: []cli.Flag{
// Global flags
NoColorFlag,
DebugFlag,
LogLevelFlag,
ExperimentsFlag,
ProfileFlag,
},
Action: func(c *cli.Context) error {
// The configuration will be loaded into this struct
cfg := PipelineEvalConfig{}

loader := cliconfig.Loader{CLI: c, Config: &cfg}
warnings, err := loader.Load()
if err != nil {
fmt.Printf("%s", err)
os.Exit(1)
}

l := CreateLogger(&cfg)

// Now that we have a logger, log out the warnings that loading config generated
for _, warning := range warnings {
l.Warn("%s", warning)
}

// Setup any global configuration options
done := HandleGlobalFlags(l, cfg)
defer done()

// Find the pipeline file either from STDIN or the first
// argument
var input []byte
var filename string

if cfg.FilePath != "" {
l.Info("Reading pipeline config from \"%s\"", cfg.FilePath)

filename = filepath.Base(cfg.FilePath)
input, err = os.ReadFile(cfg.FilePath)
if err != nil {
l.Fatal("Failed to read file: %s", err)
}
} else if stdin.IsReadable() {
l.Info("Reading pipeline config from STDIN")

// Actually read the file from STDIN
input, err = io.ReadAll(os.Stdin)
if err != nil {
l.Fatal("Failed to read from STDIN: %s", err)
}
} else {
l.Info("Searching for pipeline config...")

paths := []string{
"buildkite.js",
filepath.FromSlash(".buildkite/buildkite.js"),
filepath.FromSlash("buildkite/buildkite.js"),
}

// Collect all the files that exist
exists := []string{}
for _, path := range paths {
if _, err := os.Stat(path); err == nil {
exists = append(exists, path)
}
}

// If more than 1 of the config files exist, throw an
// error. There can only be one!!
if len(exists) > 1 {
l.Fatal("Found multiple configuration files: %s. Please only have 1 configuration file present.", strings.Join(exists, ", "))
} else if len(exists) == 0 {
l.Fatal("Could not find a default pipeline configuration file. See `buildkite-agent pipeline upload --help` for more information.")
}

found := exists[0]

l.Info("Found config file \"%s\"", found)

// Read the default file
filename = path.Base(found)
input, err = os.ReadFile(found)
if err != nil {
l.Fatal("Failed to read file \"%s\" (%s)", found, err)
}
}

if err := evalJS(filename, input, c.App.Writer); err != nil {
panic(err)
}
return nil
},
}

func evalJS(filename string, input []byte, output io.Writer) error {
runtime := goja.New()
runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true))

// Add support for require() CommonJS modules.
// require("buildkite/*") is handled by embedded resources/node_modules/buildkite/* filesystem.
// Other paths are loaded from the host filesystem.
registry := require.NewRegistry(
require.WithLoader(func(name string) ([]byte, error) {
if !strings.HasPrefix(name, "node_modules/buildkite/") {
return require.DefaultSourceLoader(name)
}
res := resources.FS
data, err := res.ReadFile(name)
if errors.Is(err, fs.ErrNotExist) {
return nil, require.ModuleFileDoesNotExistError
} else if err != nil {
return nil, err
}
return data, nil
}),
)
registry.Enable(runtime)

// Add basic utilities
console.Enable(runtime) // console.log()
process.Enable(runtime) // process.env()

// provide plugin() as a native module (implemented in Go)
registry.RegisterNativeModule("buildkite/plugin", func(runtime *goja.Runtime, module *goja.Object) {
module.Set("exports", func(call goja.FunctionCall) goja.Value {
name := call.Argument(0)
ref := call.Argument(1)
config := call.Argument(2)
plugin := runtime.NewObject()
plugin.Set(name.String()+"#"+ref.String(), config)
return plugin
})
})

// provide assignable module.exports for Pipeline result
rootModule := runtime.NewObject()
rootModule.Set("exports", runtime.NewObject())
err := runtime.Set("module", rootModule)
if err != nil {
return err
}

if filename == "" {
filename = "(stdin)"
}

v, err := runtime.RunScript(filename, string(input))
if err != nil {
panic(err)
}

y, err := yaml.Marshal(v.Export())
if err != nil {
return err
}

n, err := output.Write(y)
if err != nil {
return nil
}
if n != len(y) {
return errors.New("short write")
}

return nil
}
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,13 @@ require (
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgraph-io/ristretto v0.1.0 // indirect
github.com/dlclark/regexp2 v1.7.0 // indirect
github.com/dop251/goja v0.0.0-20221106173738-3b8a68ca89b4 // indirect
github.com/dop251/goja_nodejs v0.0.0-20221009164102-3aa5028e57f6 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.5.2 // indirect
Expand Down
22 changes: 22 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand All @@ -116,6 +117,17 @@ github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/Lu
github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo=
github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
github.com/dop251/goja v0.0.0-20220815083517-0c74f9139fd6/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs=
github.com/dop251/goja v0.0.0-20221106173738-3b8a68ca89b4 h1:arM6Tq1Ba+a9FWuq3S6Qgrfd5MD0slQdMnCKI2VclFg=
github.com/dop251/goja v0.0.0-20221106173738-3b8a68ca89b4/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs=
github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
github.com/dop251/goja_nodejs v0.0.0-20221009164102-3aa5028e57f6 h1:p3QZwRRfCN7Qr3GNBTMKBkLFjEm3DHR4MaJABvsiqgk=
github.com/dop251/goja_nodejs v0.0.0-20221009164102-3aa5028e57f6/go.mod h1:+CJy9V5cGycP5qwp6RM5jLg+TFEMyGtD7A9xUbU/BOQ=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
Expand All @@ -140,6 +152,8 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
Expand Down Expand Up @@ -251,9 +265,13 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53 h1:tGfIHhDghvEnneeRhODvGYOt305TPwingKt6p90F4MU=
Expand Down Expand Up @@ -285,6 +303,7 @@ github.com/rjeczalik/interfaces v0.1.1 h1:xhFQNGtz3T3CQgtJJwWn+i3Ekl1WeObh7wtTtC
github.com/rjeczalik/interfaces v0.1.1/go.mod h1:TNwD+kCGmXYrXksRDD5ikspp08m/Aosbr67zVLMjnOY=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sasha-s/go-deadlock v0.0.0-20180226215254-237a9547c8a5 h1:T7hUw7pBSINuHQyWwMdfIWZZH5M3ju4yXIbuV/Upp+4=
Expand Down Expand Up @@ -811,11 +830,14 @@ gopkg.in/DataDog/dd-trace-go.v1 v1.43.1/go.mod h1:YL9g+nlUY7ByCffD5pDytAqy99GNby
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
Expand Down
1 change: 1 addition & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ func main() {
Usage: "Make changes to the pipeline of the currently running build",
Subcommands: []cli.Command{
clicommand.PipelineUploadCommand,
clicommand.PipelineEvalCommand,
},
},
{
Expand Down
1 change: 1 addition & 0 deletions resources/node_modules/buildkite/hello.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions resources/resources.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package resources

import "embed"

// FS is an embedded filesystem.
//go:embed node_modules
var FS embed.FS
26 changes: 26 additions & 0 deletions test/fixtures/pipelines/buildkite.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
plugin = require("buildkite/plugin");
require("buildkite/hello");

const dockerCompose = plugin("docker-compose", "v3.0.0", {
config: ".buildkite/docker-compose.yml",
run: "agent",
});

pipeline = {
env: {
DRY_RUN: !!process.env.DRY_RUN,
},
agents: {
queue: "agent-runners-linux-amd64",
},
steps: [
{
name: ":go: go fmt",
key: "test-go-fmt",
command: ".buildkite/steps/test-go-fmt.sh",
plugins: [dockerCompose],
},
],
};

module.exports = pipeline;

0 comments on commit 063c4e4

Please sign in to comment.