Skip to content

Commit

Permalink
Buildpack system for rpk wasm build
Browse files Browse the repository at this point in the history 
Signed-off-by: Tyler Rockwood <[email protected]>
  • Loading branch information
@rockwotj
rockwotj committed Jun 23, 2023
1 parent b424441 commit 453babf
Show file tree
Hide file tree
Showing 9 changed files with 308 additions and 203 deletions.
8 changes: 7 additions & 1 deletion src/go/rpk/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ require (
github.com/avast/retry-go v3.0.0+incompatible
github.com/aws/aws-sdk-go v1.44.259
github.com/beevik/ntp v1.0.0
github.com/cespare/xxhash v1.1.0
github.com/coreos/go-systemd/v22 v22.5.0
github.com/docker/docker v23.0.6+incompatible
github.com/docker/go-connections v0.4.0
Expand Down Expand Up @@ -47,6 +46,12 @@ require (
k8s.io/client-go v0.27.1
)

require (
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/rivo/uniseg v0.2.0 // indirect
)

require (
cloud.google.com/go/compute v1.19.2 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
Expand Down Expand Up @@ -92,6 +97,7 @@ require (
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/schollz/progressbar/v3 v3.13.1
github.com/tklauser/numcpus v0.6.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
Expand Down
14 changes: 11 additions & 3 deletions src/go/rpk/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s=
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
Expand All @@ -64,7 +63,6 @@ github.com/beevik/ntp v1.0.0/go.mod h1:JN7/74B0Z4GUGO/1aUeRI2adARlfJGUeaJb0y0Wvn
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
Expand Down Expand Up @@ -223,6 +221,7 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
Expand Down Expand Up @@ -262,13 +261,18 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
Expand Down Expand Up @@ -301,15 +305,18 @@ github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUo
github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
github.com/prometheus/common v0.43.0 h1:iq+BVjvYLei5f27wiuNiB1DN6DYQkp1c8Bx0Vykh5us=
github.com/prometheus/common v0.43.0/go.mod h1:NCvr5cQIh3Y/gy73/RdVtC9r8xxrxwJnB+2lB3BxrFc=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0=
github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs=
github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE=
github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ=
github.com/sethgrid/pester v1.2.0 h1:adC9RS29rRUef3rIKWPOuP1Jm3/MmB6ke+OhE5giENI=
github.com/sethgrid/pester v1.2.0/go.mod h1:hEUINb4RqvDxtoCaU0BNT/HV4ig5kfgOasrf1xcvr0A=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM=
github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
Expand Down Expand Up @@ -545,6 +552,7 @@ golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
Expand Down
188 changes: 12 additions & 176 deletions src/go/rpk/pkg/cli/transform/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,129 +11,14 @@ package transform
import (
"context"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"

"github.com/redpanda-data/redpanda/src/go/rpk/pkg/config"
rpkos "github.com/redpanda-data/redpanda/src/go/rpk/pkg/os"
"github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/transform/buildpack"
"github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/transform/project"
"github.com/redpanda-data/redpanda/src/go/rpk/pkg/out"
"github.com/redpanda-data/redpanda/src/go/rpk/pkg/plugin"
"github.com/spf13/afero"
"github.com/spf13/cobra"
)

type buildPlugin struct {
// the name of the plugin, this will also be the command name.
Name string
// the base url to download the plugin from, the end should be `/{name}-{goos}-{goarch}.tar.gz`
baseUrl string
// [GOOS][GOARCH] = shasum
shaSums map[string]map[string]string
// A callback to modify arguments before executing the command
modifyArgs func(c *cobra.Command, p WasmProjectConfig, args []string) []string
// Other hidden plugins that are required by this plugin
deps []buildPlugin
}

func (bp *buildPlugin) ModifyArgs(c *cobra.Command, p WasmProjectConfig, args []string) []string {
if bp.modifyArgs != nil {
return bp.modifyArgs(c, p, args)
}
return args
}

func (bp *buildPlugin) Deps() []buildPlugin {
if bp.deps == nil {
return []buildPlugin{}
}
return bp.deps
}

func (bp *buildPlugin) PlatformSha() (sha string, err error) {
a, ok := bp.shaSums[runtime.GOOS]
if ok {
archSha, ok := a[runtime.GOARCH]
if ok {
sha = archSha
}
}
if sha == "" {
err = fmt.Errorf("%s plugin is not supported in this environment", bp.Name)
}
return
}

func (bp *buildPlugin) Url() string {
return fmt.Sprintf(
"%s/%s-%s-%s.tar.gz",
bp.baseUrl,
bp.Name,
runtime.GOOS,
runtime.GOARCH,
)
}

var (
tinygoPlugin = buildPlugin{
Name: "tinygo",
baseUrl: "https://github.com/rockwotj/tinygo/releases/download/v0.28.1-rpk",
shaSums: map[string]map[string]string{
"linux": map[string]string{
"amd64": "f2dff481f71319007f7715d61cabf60643838dd7ceeda05d7af68811d13e813d",
"arm64": "e42e393e073b4955eb3b25f630dfd0d013328e8d7664bc9efe97c76b833cf811",
},
"darwin": map[string]string{
"amd64": "923130c6d524d98a9521336acb486d596a1ee28861c9c28a91fdcc9902349425",
"arm64": "11714ef300a4f236b211753d4c58bf60c3400a163f599af023b1ea52835f12fe",
},
},
modifyArgs: func(c *cobra.Command, p WasmProjectConfig, args []string) []string {
// Add the output flag if not specified
for _, arg := range args {
if arg == "-o" || strings.HasPrefix(arg, "-o=") {
return args
}
}
return append(args, "-o", fmt.Sprintf("%s.wasm", p.Name))
},
deps: []buildPlugin{
{
Name: "wasm-opt",
baseUrl: "https://github.com/rockwotj/tinygo/releases/download/v0.28.1-rpk",
shaSums: map[string]map[string]string{
"linux": map[string]string{
"amd64": "e43bc07ec16e4d1758aaeaaf5b2bbe820760d87fdcdb9fbcd235818aa4add107",
"arm64": "5a1642783a57d233b5050ace12a57801652878d8f9b8c7d9fc1be4cf84c649c1",
},
"darwin": map[string]string{
"amd64": "a233945f05c85edbafa4ce39db002f284aad9e143278b8626aa1dbc4a0fcfe97",
"arm64": "daa18a3e6fec1f6d5b45c3e66d4e153ca7948212a5bf53e0b565bc7341102734",
},
},
},
},
}
)

func init() {
for _, p := range []buildPlugin{tinygoPlugin} {
plugin.RegisterManaged(p.Name, []string{"transform", "build", p.Name}, func(c *cobra.Command, fs afero.Fs, _ *config.Params) *cobra.Command {
run := c.Run
c.Run = func(cmd *cobra.Command, args []string) {
cfg, err := loadCfg(fs)
out.MaybeDie(err, "unable to find the transform, are you in the same directory as the %q?", configFileName)
_, err = installPlugin(cmd.Context(), tinygoPlugin, fs)
out.MaybeDie(err, "unable to install %s plugin: %v", p.Name, err)
args = p.ModifyArgs(cmd, cfg, args)
run(cmd, args)
}
return c
})
}
}

func newBuildCommand(fs afero.Fs, execFn func(string, []string) error) *cobra.Command {
cmd := &cobra.Command{
Use: "build",
Expand All @@ -147,11 +32,11 @@ build a .wasm file.
`,
Args: cobra.NoArgs,
Run: func(cmd *cobra.Command, args []string) {
cfg, err := loadCfg(fs)
out.MaybeDie(err, "unable to find the transform, are you in the same directory as the %q?", configFileName)
cfg, err := project.LoadCfg(fs)
out.MaybeDie(err, "unable to find the transform, are you in the same directory as the %q?", project.ConfigFileName)
switch cfg.Language {
case WasmLangTinygo:
tinygo, err := installPlugin(cmd.Context(), tinygoPlugin, fs)
case project.WasmLangTinygo:
tinygo, err := installBuildpack(cmd.Context(), buildpack.Tinygo, fs)
out.MaybeDie(err, "unable to install tinygo plugin: %v", err)
args := []string{"-o", fmt.Sprintf("%s.wasm", cfg.Name)}
out.MaybeDieErr(execFn(tinygo, args))
Expand All @@ -163,68 +48,19 @@ build a .wasm file.
return cmd
}

func installPlugin(ctx context.Context, p buildPlugin, fs afero.Fs) (path string, err error) {
sha, err := p.PlatformSha()
func installBuildpack(ctx context.Context, p buildpack.Buildpack, fs afero.Fs) (path string, err error) {
ok, err := p.IsUpToDate(fs)
if err != nil {
return "", err
}
pluginDir, err := plugin.DefaultBinPath()
if err != nil {
return "", fmt.Errorf("unable to determine managed plugin path: %v", err)
}
tinygo, pluginExists := plugin.ListPlugins(fs, []string{pluginDir}).Find(p.Name)
if pluginExists {
fsha, err := plugin.Sha256Path(fs, tinygo.Path)
if err != nil {
return "", fmt.Errorf("unable to determine if plugin is up to date: %v", err)
}
// TODO: We should verify that deps are present and the right shas
if sha == fsha {
return tinygo.Path, nil
}
if ok {
return p.BinPath()
}
fmt.Printf("latest %s build plugin not found, downloading now...\n", p.Name)
path, err = downloadPlugin(ctx, p, fs, sha, false)
err = p.Download(ctx, fs)
if err != nil {
return "", err
}
for _, dep := range p.Deps() {
depSha, err := dep.PlatformSha()
if err != nil {
return "", err
}
_, err = downloadPlugin(ctx, dep, fs, depSha, true)
if err != nil {
return "", err
}
}
fmt.Printf("latest %s build plugin download complete\n", p.Name)
return path, nil
}

func downloadPlugin(ctx context.Context, p buildPlugin, fs afero.Fs, expectedSha string, dep bool) (path string, err error) {
pluginDir, err := plugin.DefaultBinPath()
if err != nil {
return "", fmt.Errorf("unable to determine managed plugin path: %v", err)
}
bin, err := plugin.Download(ctx, p.Url(), true, expectedSha)
if exists, _ := afero.DirExists(fs, pluginDir); !exists {
if rpkos.IsRunningSudo() {
return "", fmt.Errorf("detected rpk is running with sudo; please execute this command without sudo to avoid saving the plugin as a root owned binary in %s", pluginDir)
}
err = os.MkdirAll(pluginDir, 0o755)
if err != nil {
return "", fmt.Errorf("unable to create plugin bin directory: %v", err)
}
}
if dep {
path = filepath.Join(pluginDir, p.Name)
err = rpkos.ReplaceFile(fs, path, bin, 0o755)
} else {
path, err = plugin.WriteBinary(fs, p.Name, pluginDir, bin, false, true)
}
if err != nil {
return "", fmt.Errorf("unable to write %s plugin to disk: %v", p.Name, err)
}
return path, nil
return p.BinPath()
}
Loading

0 comments on commit 453babf

Please sign in to comment.