diff --git a/cmd/cli/config/set.go b/cmd/cli/config/set.go index 3f9a3a669f..1e885824a3 100644 --- a/cmd/cli/config/set.go +++ b/cmd/cli/config/set.go @@ -10,12 +10,13 @@ import ( "github.com/spf13/viper" "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags" "github.com/bacalhau-project/bacalhau/cmd/util/hook" "github.com/bacalhau-project/bacalhau/pkg/config" "github.com/bacalhau-project/bacalhau/pkg/config/types" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" ) var setExample = templates.Examples(i18n.T(` diff --git a/cmd/cli/devstack/devstack.go b/cmd/cli/devstack/devstack.go index 80fb3ed4d2..50719a1fb4 100644 --- a/cmd/cli/devstack/devstack.go +++ b/cmd/cli/devstack/devstack.go @@ -17,10 +17,11 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/logger" "github.com/bacalhau-project/bacalhau/webui" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/pkg/devstack" "github.com/bacalhau-project/bacalhau/pkg/telemetry" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" "github.com/spf13/cobra" ) diff --git a/cmd/cli/docker/docker_run.go b/cmd/cli/docker/docker_run.go index ad861bbe73..c4bd440787 100644 --- a/cmd/cli/docker/docker_run.go +++ b/cmd/cli/docker/docker_run.go @@ -8,6 +8,8 @@ import ( "github.com/spf13/viper" "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/cli/helpers" "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags" @@ -20,7 +22,6 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels" clientv2 "github.com/bacalhau-project/bacalhau/pkg/publicapi/client/v2" "github.com/bacalhau-project/bacalhau/pkg/userstrings" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" ) var ( diff --git a/cmd/cli/job/describe.go b/cmd/cli/job/describe.go index 16f38a7655..4ecc5c6a1f 100644 --- a/cmd/cli/job/describe.go +++ b/cmd/cli/job/describe.go @@ -16,11 +16,12 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/publicapi/client/v2" "github.com/bacalhau-project/bacalhau/pkg/util/idgen" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags" "github.com/bacalhau-project/bacalhau/cmd/util/output" "github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" ) var ( diff --git a/cmd/cli/job/executions.go b/cmd/cli/job/executions.go index 428262a8eb..d3c3f162ef 100644 --- a/cmd/cli/job/executions.go +++ b/cmd/cli/job/executions.go @@ -11,6 +11,8 @@ import ( "github.com/spf13/cobra" "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags" "github.com/bacalhau-project/bacalhau/cmd/util/output" @@ -18,7 +20,6 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels" "github.com/bacalhau-project/bacalhau/pkg/publicapi/client/v2" "github.com/bacalhau-project/bacalhau/pkg/util/idgen" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" ) var executionsOrderByFields = []string{"modified_at", "created_at"} diff --git a/cmd/cli/job/get.go b/cmd/cli/job/get.go index 93a5a028b7..35515fbdf3 100644 --- a/cmd/cli/job/get.go +++ b/cmd/cli/job/get.go @@ -8,13 +8,14 @@ import ( "github.com/spf13/viper" "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags" "github.com/bacalhau-project/bacalhau/cmd/util/flags/configflags" "github.com/bacalhau-project/bacalhau/cmd/util/hook" "github.com/bacalhau-project/bacalhau/pkg/config/types" "github.com/bacalhau-project/bacalhau/pkg/publicapi/client/v2" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" ) var ( diff --git a/cmd/cli/job/history.go b/cmd/cli/job/history.go index f78abf1242..90da87e6e8 100644 --- a/cmd/cli/job/history.go +++ b/cmd/cli/job/history.go @@ -7,6 +7,8 @@ import ( "github.com/spf13/cobra" "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/cmd/util/cols" "github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags" @@ -14,7 +16,6 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/models" "github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels" "github.com/bacalhau-project/bacalhau/pkg/publicapi/client/v2" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" ) var ( diff --git a/cmd/cli/job/list.go b/cmd/cli/job/list.go index 03806ab730..bf3296c251 100644 --- a/cmd/cli/job/list.go +++ b/cmd/cli/job/list.go @@ -10,6 +10,8 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags" "github.com/bacalhau-project/bacalhau/cmd/util/output" @@ -18,7 +20,6 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels" "github.com/bacalhau-project/bacalhau/pkg/publicapi/client/v2" "github.com/bacalhau-project/bacalhau/pkg/util/idgen" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" ) var orderByFields = []string{"id", "created_at"} diff --git a/cmd/cli/job/logs.go b/cmd/cli/job/logs.go index 016e58c25f..839d474eb5 100644 --- a/cmd/cli/job/logs.go +++ b/cmd/cli/job/logs.go @@ -6,8 +6,9 @@ import ( "github.com/spf13/cobra" "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/util" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" ) var ( diff --git a/cmd/cli/job/run.go b/cmd/cli/job/run.go index ee63a72972..dfaf8765dc 100644 --- a/cmd/cli/job/run.go +++ b/cmd/cli/job/run.go @@ -12,11 +12,12 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels" "github.com/bacalhau-project/bacalhau/pkg/publicapi/client/v2" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags" "github.com/bacalhau-project/bacalhau/cmd/util/printer" "github.com/bacalhau-project/bacalhau/pkg/userstrings" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" ) var ( diff --git a/cmd/cli/job/stop.go b/cmd/cli/job/stop.go index e7a1ad6a23..1aad3f4864 100644 --- a/cmd/cli/job/stop.go +++ b/cmd/cli/job/stop.go @@ -10,10 +10,11 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels" "github.com/bacalhau-project/bacalhau/pkg/publicapi/client/v2" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/cmd/util/printer" "github.com/bacalhau-project/bacalhau/pkg/bacerrors" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" ) var ( diff --git a/cmd/cli/job/validate.go b/cmd/cli/job/validate.go index 3163cc68ec..ae3d9bea76 100644 --- a/cmd/cli/job/validate.go +++ b/cmd/cli/job/validate.go @@ -4,10 +4,11 @@ import ( "github.com/spf13/cobra" "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/pkg/lib/marshaller" "github.com/bacalhau-project/bacalhau/pkg/userstrings" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" ) var ( diff --git a/cmd/cli/serve/serve.go b/cmd/cli/serve/serve.go index e2bdb8a2f3..33d679c8fd 100644 --- a/cmd/cli/serve/serve.go +++ b/cmd/cli/serve/serve.go @@ -11,6 +11,8 @@ import ( "github.com/spf13/viper" "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/cmd/util/flags/configflags" "github.com/bacalhau-project/bacalhau/pkg/analytics" @@ -23,7 +25,6 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/repo" "github.com/bacalhau-project/bacalhau/pkg/setup" "github.com/bacalhau-project/bacalhau/pkg/system" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" "github.com/bacalhau-project/bacalhau/pkg/version" "github.com/bacalhau-project/bacalhau/webui" ) diff --git a/cmd/cli/serve/serve_test.go b/cmd/cli/serve/serve_test.go index a4b161e5c1..22219ce8b9 100644 --- a/cmd/cli/serve/serve_test.go +++ b/cmd/cli/serve/serve_test.go @@ -23,13 +23,11 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/setup" "github.com/bacalhau-project/bacalhau/pkg/docker" - "github.com/bacalhau-project/bacalhau/pkg/lib/marshaller" "github.com/bacalhau-project/bacalhau/pkg/lib/network" cmd2 "github.com/bacalhau-project/bacalhau/cmd/cli" "github.com/bacalhau-project/bacalhau/pkg/logger" "github.com/bacalhau-project/bacalhau/pkg/system" - "github.com/bacalhau-project/bacalhau/pkg/types" "github.com/bacalhau-project/bacalhau/pkg/util/closer" ) @@ -170,16 +168,6 @@ func (s *ServeSuite) curlEndpoint(URL string) ([]byte, int, error) { } return responseText, resp.StatusCode, nil } -func (s *ServeSuite) TestHealthcheck() { - port, _ := s.serve() - healthzText, statusCode, err := s.curlEndpoint(fmt.Sprintf("http://127.0.0.1:%d/api/v1/healthz", port)) - s.Require().NoError(err) - - var healthzJSON types.HealthInfo - s.Require().NoError(marshaller.JSONUnmarshalWithMax(healthzText, &healthzJSON), "Error unmarshalling healthz JSON.") - s.Require().Greater(int(healthzJSON.DiskFreeSpace.ROOT.All), 0, "Did not report DiskFreeSpace > 0.") - s.Require().Equal(http.StatusOK, statusCode, "Did not return 200 OK.") -} func (s *ServeSuite) TestCanSubmitJob() { docker.MustHaveDocker(s.T()) diff --git a/cmd/cli/wasm/wasm_run.go b/cmd/cli/wasm/wasm_run.go index e01e0697d0..29e07e23af 100644 --- a/cmd/cli/wasm/wasm_run.go +++ b/cmd/cli/wasm/wasm_run.go @@ -15,6 +15,8 @@ import ( "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" + "github.com/bacalhau-project/bacalhau/cmd/cli/helpers" "github.com/bacalhau-project/bacalhau/cmd/util" "github.com/bacalhau-project/bacalhau/cmd/util/flags" @@ -32,7 +34,6 @@ import ( storage_ipfs "github.com/bacalhau-project/bacalhau/pkg/storage/ipfs" "github.com/bacalhau-project/bacalhau/pkg/userstrings" "github.com/bacalhau-project/bacalhau/pkg/util/closer" - "github.com/bacalhau-project/bacalhau/pkg/util/templates" ) var ( diff --git a/pkg/executor/wasm/executor.go b/pkg/executor/wasm/executor.go index 026069e709..2795a8cf39 100644 --- a/pkg/executor/wasm/executor.go +++ b/pkg/executor/wasm/executor.go @@ -15,17 +15,17 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/lib/math" "github.com/bacalhau-project/bacalhau/pkg/models" "github.com/bacalhau-project/bacalhau/pkg/telemetry" + "github.com/bacalhau-project/bacalhau/pkg/util/generic" "github.com/bacalhau-project/bacalhau/pkg/bidstrategy" "github.com/bacalhau-project/bacalhau/pkg/executor" wasmmodels "github.com/bacalhau-project/bacalhau/pkg/executor/wasm/models" - wasmlogs "github.com/bacalhau-project/bacalhau/pkg/logger/wasm" + "github.com/bacalhau-project/bacalhau/pkg/executor/wasm/util/filefs" + wasmlogs "github.com/bacalhau-project/bacalhau/pkg/executor/wasm/util/logger" + "github.com/bacalhau-project/bacalhau/pkg/executor/wasm/util/mountfs" + "github.com/bacalhau-project/bacalhau/pkg/executor/wasm/util/touchfs" "github.com/bacalhau-project/bacalhau/pkg/storage" "github.com/bacalhau-project/bacalhau/pkg/storage/util" - "github.com/bacalhau-project/bacalhau/pkg/util/filefs" - "github.com/bacalhau-project/bacalhau/pkg/util/generic" - "github.com/bacalhau-project/bacalhau/pkg/util/mountfs" - "github.com/bacalhau-project/bacalhau/pkg/util/touchfs" ) type Executor struct { diff --git a/pkg/executor/wasm/handler.go b/pkg/executor/wasm/handler.go index 2acb054f09..6ed219623e 100644 --- a/pkg/executor/wasm/handler.go +++ b/pkg/executor/wasm/handler.go @@ -18,7 +18,7 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/executor" wasmmodels "github.com/bacalhau-project/bacalhau/pkg/executor/wasm/models" - wasmlogs "github.com/bacalhau-project/bacalhau/pkg/logger/wasm" + wasmlogs "github.com/bacalhau-project/bacalhau/pkg/executor/wasm/util/logger" "github.com/bacalhau-project/bacalhau/pkg/models" "github.com/bacalhau-project/bacalhau/pkg/storage" "github.com/bacalhau-project/bacalhau/pkg/telemetry" diff --git a/pkg/util/filefs/fs.go b/pkg/executor/wasm/util/filefs/fs.go similarity index 58% rename from pkg/util/filefs/fs.go rename to pkg/executor/wasm/util/filefs/fs.go index 93ba809615..91e74f80fb 100644 --- a/pkg/util/filefs/fs.go +++ b/pkg/executor/wasm/util/filefs/fs.go @@ -19,17 +19,3 @@ func (f fileFs) Open(name string) (fs.File, error) { return nil, os.ErrNotExist } } - -func IsWritable(path string) (bool, error) { - tmpFile := "tmpfile" - - file, err := os.CreateTemp(path, tmpFile) - if err != nil { - return false, err - } - - defer os.Remove(file.Name()) - defer file.Close() - - return true, nil -} diff --git a/pkg/util/generic/broadcaster.go b/pkg/executor/wasm/util/generic/broadcaster.go similarity index 100% rename from pkg/util/generic/broadcaster.go rename to pkg/executor/wasm/util/generic/broadcaster.go diff --git a/pkg/util/generic/broadcaster_test.go b/pkg/executor/wasm/util/generic/broadcaster_test.go similarity index 95% rename from pkg/util/generic/broadcaster_test.go rename to pkg/executor/wasm/util/generic/broadcaster_test.go index ca10bf067d..42be342077 100644 --- a/pkg/util/generic/broadcaster_test.go +++ b/pkg/executor/wasm/util/generic/broadcaster_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/bacalhau-project/bacalhau/pkg/executor/wasm/util/generic" _ "github.com/bacalhau-project/bacalhau/pkg/logger" - "github.com/bacalhau-project/bacalhau/pkg/util/generic" ) type BroadcasterTestSuite struct { diff --git a/pkg/util/generic/ringbuffer.go b/pkg/executor/wasm/util/generic/ringbuffer.go similarity index 100% rename from pkg/util/generic/ringbuffer.go rename to pkg/executor/wasm/util/generic/ringbuffer.go diff --git a/pkg/util/generic/ringbuffer_test.go b/pkg/executor/wasm/util/generic/ringbuffer_test.go similarity index 95% rename from pkg/util/generic/ringbuffer_test.go rename to pkg/executor/wasm/util/generic/ringbuffer_test.go index d75514ef56..e34964453e 100644 --- a/pkg/util/generic/ringbuffer_test.go +++ b/pkg/executor/wasm/util/generic/ringbuffer_test.go @@ -6,10 +6,11 @@ import ( "testing" "time" - _ "github.com/bacalhau-project/bacalhau/pkg/logger" - "github.com/bacalhau-project/bacalhau/pkg/util/generic" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + + "github.com/bacalhau-project/bacalhau/pkg/executor/wasm/util/generic" + _ "github.com/bacalhau-project/bacalhau/pkg/logger" ) type RingBufferTestSuite struct { diff --git a/pkg/logger/wasm/logmanager.go b/pkg/executor/wasm/util/logger/logmanager.go similarity index 98% rename from pkg/logger/wasm/logmanager.go rename to pkg/executor/wasm/util/logger/logmanager.go index b2c3735409..23a0929a9b 100644 --- a/pkg/logger/wasm/logmanager.go +++ b/pkg/executor/wasm/util/logger/logmanager.go @@ -12,9 +12,9 @@ import ( "github.com/rs/zerolog/log" + "github.com/bacalhau-project/bacalhau/pkg/executor/wasm/util/generic" "github.com/bacalhau-project/bacalhau/pkg/logger" "github.com/bacalhau-project/bacalhau/pkg/util" - "github.com/bacalhau-project/bacalhau/pkg/util/generic" ) const ( diff --git a/pkg/logger/wasm/logmanager_test.go b/pkg/executor/wasm/util/logger/logmanager_test.go similarity index 100% rename from pkg/logger/wasm/logmanager_test.go rename to pkg/executor/wasm/util/logger/logmanager_test.go diff --git a/pkg/logger/wasm/logmessage.go b/pkg/executor/wasm/util/logger/logmessage.go similarity index 100% rename from pkg/logger/wasm/logmessage.go rename to pkg/executor/wasm/util/logger/logmessage.go diff --git a/pkg/logger/wasm/logmessage_test.go b/pkg/executor/wasm/util/logger/logmessage_test.go similarity index 100% rename from pkg/logger/wasm/logmessage_test.go rename to pkg/executor/wasm/util/logger/logmessage_test.go diff --git a/pkg/logger/wasm/logreader.go b/pkg/executor/wasm/util/logger/logreader.go similarity index 98% rename from pkg/logger/wasm/logreader.go rename to pkg/executor/wasm/util/logger/logreader.go index 89305d64b0..e11057edc7 100644 --- a/pkg/logger/wasm/logreader.go +++ b/pkg/executor/wasm/util/logger/logreader.go @@ -7,8 +7,9 @@ import ( "io" "os" - "github.com/bacalhau-project/bacalhau/pkg/util/generic" "github.com/rs/zerolog/log" + + "github.com/bacalhau-project/bacalhau/pkg/executor/wasm/util/generic" ) const ( diff --git a/pkg/logger/wasm/logwriter.go b/pkg/executor/wasm/util/logger/logwriter.go similarity index 86% rename from pkg/logger/wasm/logwriter.go rename to pkg/executor/wasm/util/logger/logwriter.go index 5b8c2aee18..1f9dd51c32 100644 --- a/pkg/logger/wasm/logwriter.go +++ b/pkg/executor/wasm/util/logger/logwriter.go @@ -1,6 +1,8 @@ package wasmlogs -import "github.com/bacalhau-project/bacalhau/pkg/util/generic" +import ( + "github.com/bacalhau-project/bacalhau/pkg/executor/wasm/util/generic" +) type LogWriterTransform func([]byte) *LogMessage diff --git a/pkg/util/mountfs/dir.go b/pkg/executor/wasm/util/mountfs/dir.go similarity index 100% rename from pkg/util/mountfs/dir.go rename to pkg/executor/wasm/util/mountfs/dir.go diff --git a/pkg/util/mountfs/direntry.go b/pkg/executor/wasm/util/mountfs/direntry.go similarity index 100% rename from pkg/util/mountfs/direntry.go rename to pkg/executor/wasm/util/mountfs/direntry.go diff --git a/pkg/util/mountfs/fs.go b/pkg/executor/wasm/util/mountfs/fs.go similarity index 100% rename from pkg/util/mountfs/fs.go rename to pkg/executor/wasm/util/mountfs/fs.go diff --git a/pkg/util/mountfs/fs_test.go b/pkg/executor/wasm/util/mountfs/fs_test.go similarity index 100% rename from pkg/util/mountfs/fs_test.go rename to pkg/executor/wasm/util/mountfs/fs_test.go diff --git a/pkg/util/touchfs/fs.go b/pkg/executor/wasm/util/touchfs/fs.go similarity index 100% rename from pkg/util/touchfs/fs.go rename to pkg/executor/wasm/util/touchfs/fs.go diff --git a/pkg/util/touchfs/fs_test.go b/pkg/executor/wasm/util/touchfs/fs_test.go similarity index 100% rename from pkg/util/touchfs/fs_test.go rename to pkg/executor/wasm/util/touchfs/fs_test.go diff --git a/pkg/ipfs/client.go b/pkg/ipfs/client.go index a17e6c04d7..e329eec5d6 100644 --- a/pkg/ipfs/client.go +++ b/pkg/ipfs/client.go @@ -20,10 +20,9 @@ import ( ma "github.com/multiformats/go-multiaddr" "github.com/pkg/errors" "github.com/rs/zerolog/log" + "github.com/samber/lo" "github.com/bacalhau-project/bacalhau/pkg/system" - "github.com/bacalhau-project/bacalhau/pkg/util/generic" - "github.com/bacalhau-project/bacalhau/pkg/util/multiaddresses" ) // Client is a front-end for an ipfs node's API endpoints @@ -95,7 +94,7 @@ func (cl Client) SwarmMultiAddresses(ctx context.Context) ([]ma.Multiaddr, error return nil, fmt.Errorf("error fetching node's swarm addresses: %w", err) } - addrs = generic.Map(addrs, func(f ma.Multiaddr) ma.Multiaddr { + addrs = lo.Map(addrs, func(f ma.Multiaddr, _ int) ma.Multiaddr { return f.Encapsulate(p2pID) }) @@ -117,9 +116,9 @@ func (cl Client) SwarmAddresses(ctx context.Context) ([]string, error) { // This sorts the addresses so IPv4 localhost is first, with the aim of using the localhost connection during tests // and so avoid any unneeded network hops. Other callers to this either sort the list themselves or just output the // full list. - multiAddresses = multiaddresses.SortLocalhostFirst(multiAddresses) + multiAddresses = SortLocalhostFirst(multiAddresses) - addresses := generic.Map(multiAddresses, func(f ma.Multiaddr) string { + addresses := lo.Map(multiAddresses, func(f ma.Multiaddr, _ int) string { return f.String() }) diff --git a/pkg/ipfs/utils.go b/pkg/ipfs/utils.go index c049f8c0fa..4ffafea7de 100644 --- a/pkg/ipfs/utils.go +++ b/pkg/ipfs/utils.go @@ -5,11 +5,13 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" - "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" "github.com/pkg/errors" "github.com/rs/zerolog/log" + "golang.org/x/exp/slices" "github.com/bacalhau-project/bacalhau/pkg/storage/util" ) @@ -45,19 +47,26 @@ func AddTextToNodes(ctx context.Context, fileContent []byte, clients ...Client) return AddFileToNodes(ctx, testFilePath, clients...) } -func ParsePeersString(peers []string) ([]peer.AddrInfo, error) { - // Parse the bootstrap node multiaddrs and fetch their IPFS peer info: - var res []peer.AddrInfo - for _, p := range peers { - if p == "" { - continue +func SortLocalhostFirst(multiAddresses []multiaddr.Multiaddr) []multiaddr.Multiaddr { + multiAddresses = slices.Clone(multiAddresses) + preferLocalhost := func(m multiaddr.Multiaddr) int { + count := 0 + if _, err := m.ValueForProtocol(multiaddr.P_TCP); err == nil { + count++ } - pi, err := peer.AddrInfoFromString(p) - if err != nil { - return nil, err + if ip, err := m.ValueForProtocol(multiaddr.P_IP4); err == nil { + count++ + if ip == "127.0.0.1" { + count++ + } + } else if ip, err := m.ValueForProtocol(multiaddr.P_IP6); err == nil && ip != "::1" { + count++ } - res = append(res, *pi) + return count } + sort.Slice(multiAddresses, func(i, j int) bool { + return preferLocalhost(multiAddresses[i]) > preferLocalhost(multiAddresses[j]) + }) - return res, nil + return multiAddresses } diff --git a/pkg/jobstore/envelope.go b/pkg/jobstore/envelope.go deleted file mode 100644 index fe3bfa6fc7..0000000000 --- a/pkg/jobstore/envelope.go +++ /dev/null @@ -1,59 +0,0 @@ -package jobstore - -import "github.com/bacalhau-project/bacalhau/pkg/lib/marshaller" - -// Envelope provides a wrapper around types that can be stored in a jobstore. -// It takes responsibility for the wrapped type, ensuring that -type Envelope[T any] struct { - Body T - marshaller marshaller.Marshaller -} - -type Option[T any] func(*Envelope[T]) - -func WithBody[T any](body T) Option[T] { - return func(e *Envelope[T]) { - e.Wrap(body) - } -} - -func WithMarshaller[T any](marshaller marshaller.Marshaller) Option[T] { - return func(e *Envelope[T]) { - e.marshaller = marshaller - } -} - -func NewEnvelope[T any](options ...Option[T]) *Envelope[T] { - e := &Envelope[T]{ - marshaller: marshaller.NewJSONMarshaller(), - } - - for _, opt := range options { - opt(e) - } - return e -} - -func (e *Envelope[T]) Copy() *Envelope[T] { - return &Envelope[T]{ - marshaller: e.marshaller, - } -} - -func (e *Envelope[T]) Wrap(obj T) { - e.Body = obj -} - -func (e *Envelope[T]) Unwrap() T { - return e.Body -} - -func (e *Envelope[T]) Serialize() ([]byte, error) { - return e.marshaller.Marshal(e) -} - -func (e *Envelope[T]) Deserialize(data []byte) (*Envelope[T], error) { - env := e.Copy() - err := e.marshaller.Unmarshal(data, env) - return env, err -} diff --git a/pkg/jobstore/envelope_test.go b/pkg/jobstore/envelope_test.go deleted file mode 100644 index 488352b964..0000000000 --- a/pkg/jobstore/envelope_test.go +++ /dev/null @@ -1,54 +0,0 @@ -//go:build unit || !integration - -package jobstore - -import ( - "testing" - - "github.com/bacalhau-project/bacalhau/pkg/lib/marshaller" - "github.com/stretchr/testify/suite" -) - -type EnvelopeTestSuite struct { - suite.Suite -} - -func TestEnvelopeTestSuite(t *testing.T) { - suite.Run(t, new(EnvelopeTestSuite)) -} - -func (s *EnvelopeTestSuite) TestEnvelope() { - type test struct { - Value string - } - - e := NewEnvelope[test]( - WithBody(test{Value: "hello"}), - WithMarshaller[test](marshaller.NewJSONMarshaller()), - ) - - encoded, err := e.Serialize() - s.NoError(err) - - newEnv, err := e.Deserialize(encoded) - s.NoError(err) - s.Equal(newEnv.Body.Value, e.Body.Value) -} - -func (s *EnvelopeTestSuite) TestEnvelopeBinaryEncoding() { - type test struct { - Value string - } - - e := NewEnvelope[test]( - WithBody(test{Value: "hello"}), - WithMarshaller[test](marshaller.NewBinaryMarshaller()), - ) - - encoded, err := e.Serialize() - s.NoError(err) - - newEnv, err := e.Deserialize(encoded) - s.NoError(err) - s.Equal(newEnv.Body.Value, e.Body.Value) -} diff --git a/pkg/lib/marshaller/binary.go b/pkg/lib/marshaller/binary.go deleted file mode 100644 index 42bede1445..0000000000 --- a/pkg/lib/marshaller/binary.go +++ /dev/null @@ -1,43 +0,0 @@ -package marshaller - -import ( - "bytes" - "encoding/gob" - "fmt" -) - -// BinaryMarshaller uses gob encoding for marshaling. -type BinaryMarshaller struct{} - -// NewBinaryMarshaller initializes and returns a new BinaryMarshaller. -func NewBinaryMarshaller() *BinaryMarshaller { - return &BinaryMarshaller{} -} - -// Marshal converts the given object into a gob-encoded byte slice. -func (BinaryMarshaller) Marshal(obj interface{}) ([]byte, error) { - var b bytes.Buffer - e := gob.NewEncoder(&b) - - if err := e.Encode(obj); err != nil { - return nil, fmt.Errorf("gob encode: %w", err) - } - - return b.Bytes(), nil -} - -// Unmarshal decodes gob data into the given object. -func (BinaryMarshaller) Unmarshal(data []byte, obj interface{}) error { - b := bytes.NewBuffer(data) - d := gob.NewDecoder(b) - - if err := d.Decode(obj); err != nil { - return fmt.Errorf("gob decode: %w", err) - } - - normalizeIfApplicable(obj) - return nil -} - -// compile-time check that BinaryMarshaller implements Marshaller -var _ Marshaller = BinaryMarshaller{} diff --git a/pkg/lib/marshaller/marshaller_test.go b/pkg/lib/marshaller/marshaller_test.go index 060fecc052..747be0d665 100644 --- a/pkg/lib/marshaller/marshaller_test.go +++ b/pkg/lib/marshaller/marshaller_test.go @@ -26,7 +26,6 @@ type MockNonNormalizable struct { func TestMarshaller(t *testing.T) { marshallers := []Marshaller{ NewJSONMarshaller(), - NewBinaryMarshaller(), } for _, m := range marshallers { diff --git a/pkg/node/requester.go b/pkg/node/requester.go index f48ba7f6ee..376f9b0760 100644 --- a/pkg/node/requester.go +++ b/pkg/node/requester.go @@ -9,6 +9,8 @@ import ( "go.opentelemetry.io/otel/attribute" "github.com/bacalhau-project/bacalhau/pkg/bacerrors" + "github.com/bacalhau-project/bacalhau/pkg/compute" + "github.com/bacalhau-project/bacalhau/pkg/jobstore" boltjobstore "github.com/bacalhau-project/bacalhau/pkg/jobstore/boltdb" "github.com/bacalhau-project/bacalhau/pkg/lib/ncl" "github.com/bacalhau-project/bacalhau/pkg/models" @@ -21,6 +23,8 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/orchestrator/planner" "github.com/bacalhau-project/bacalhau/pkg/orchestrator/retry" "github.com/bacalhau-project/bacalhau/pkg/orchestrator/scheduler" + "github.com/bacalhau-project/bacalhau/pkg/orchestrator/selection/discovery" + "github.com/bacalhau-project/bacalhau/pkg/orchestrator/selection/ranking" "github.com/bacalhau-project/bacalhau/pkg/orchestrator/selection/selector" "github.com/bacalhau-project/bacalhau/pkg/orchestrator/transformer" "github.com/bacalhau-project/bacalhau/pkg/publicapi" @@ -32,12 +36,6 @@ import ( "github.com/bacalhau-project/bacalhau/pkg/routing/tracing" s3helper "github.com/bacalhau-project/bacalhau/pkg/s3" "github.com/bacalhau-project/bacalhau/pkg/system" - "github.com/bacalhau-project/bacalhau/pkg/util" - - "github.com/bacalhau-project/bacalhau/pkg/compute" - "github.com/bacalhau-project/bacalhau/pkg/jobstore" - "github.com/bacalhau-project/bacalhau/pkg/orchestrator/selection/discovery" - "github.com/bacalhau-project/bacalhau/pkg/orchestrator/selection/ranking" ) var ( @@ -263,7 +261,7 @@ func NewRequesterNode( // close the ncl subscriber cleanupErr := subscriber.Close(ctx) if cleanupErr != nil { - util.LogDebugIfContextCancelled(ctx, cleanupErr, "failed to cleanly shutdown ncl subscriber") + logDebugIfContextCancelled(ctx, cleanupErr, "failed to cleanly shutdown ncl subscriber") } // stop the housekeeping background task @@ -276,7 +274,7 @@ func NewRequesterNode( // Close the jobstore after the evaluation broker is disabled cleanupErr = jobStore.Close(ctx) if cleanupErr != nil { - util.LogDebugIfContextCancelled(ctx, cleanupErr, "failed to cleanly shutdown jobstore") + logDebugIfContextCancelled(ctx, cleanupErr, "failed to cleanly shutdown jobstore") } } diff --git a/pkg/node/utils.go b/pkg/node/utils.go index 28673152d7..05e72aca00 100644 --- a/pkg/node/utils.go +++ b/pkg/node/utils.go @@ -95,3 +95,14 @@ func scaleCapacityByAllocation(systemCapacity models.Resources, scaler types.Res return *allocatedCapacity, nil } + +func logDebugIfContextCancelled(ctx context.Context, cleanupErr error, msg string) { + if cleanupErr == nil { + return + } + if !errors.Is(cleanupErr, context.Canceled) { + log.Ctx(ctx).Error().Err(cleanupErr).Msg("failed to close " + msg) + } else { + log.Ctx(ctx).Debug().Err(cleanupErr).Msgf("Context canceled: %s", msg) + } +} diff --git a/pkg/publicapi/endpoint/shared/endpoint.go b/pkg/publicapi/endpoint/shared/endpoint.go index ff0515c6be..bbd22f2452 100644 --- a/pkg/publicapi/endpoint/shared/endpoint.go +++ b/pkg/publicapi/endpoint/shared/endpoint.go @@ -34,7 +34,6 @@ func NewEndpoint(params EndpointParams) *Endpoint { g.Use(middleware.SetContentType(echo.MIMEApplicationJSON)) g.GET("/node_info", e.nodeInfo) g.POST("/version", e.version) - g.GET("/healthz", e.healthz) // Plaintext group pt := e.router.Group("/api/v1") @@ -111,20 +110,6 @@ func (e *Endpoint) version(c echo.Context) error { }) } -// healthz godoc -// -// @ID healthz -// @Tags Utils -// @Produce json -// @Success 200 {object} types.HealthInfo -// @Router /api/v1/healthz [get] -func (e *Endpoint) healthz(c echo.Context) error { - // TODO: A list of health information. Should require authing (of some kind) - // Ideas: - // CPU usage - return c.JSON(http.StatusOK, GenerateHealthData()) -} - // livez godoc // // @ID livez diff --git a/pkg/publicapi/endpoint/shared/utils.go b/pkg/publicapi/endpoint/shared/utils.go deleted file mode 100644 index c0461fc9ac..0000000000 --- a/pkg/publicapi/endpoint/shared/utils.go +++ /dev/null @@ -1,31 +0,0 @@ -package shared - -import ( - "github.com/ricochet2200/go-disk-usage/du" - - "github.com/bacalhau-project/bacalhau/pkg/types" -) - -func GenerateHealthData() types.HealthInfo { - var healthInfo types.HealthInfo - - // Generating all, free, used amounts for each - in case these are different mounts, they'll have different - // All and Free values, if they're all on the same machine, then those values should be the same - // If "All" is 0, that means the directory does not exist - healthInfo.DiskFreeSpace.ROOT = MountUsage("/") - healthInfo.DiskFreeSpace.TMP = MountUsage("/tmp") - - return healthInfo -} - -// Function to get disk usage of path/disk -func MountUsage(path string) (disk types.MountStatus) { - usage := du.NewDiskUsage(path) - if usage == nil { - return - } - disk.All = usage.Size() - disk.Free = usage.Free() - disk.Used = usage.Used() - return -} diff --git a/pkg/types/types.go b/pkg/types/types.go deleted file mode 100644 index 8ea343de7a..0000000000 --- a/pkg/types/types.go +++ /dev/null @@ -1,31 +0,0 @@ -package types - -// TODO: migrate all of these API types to publicapi - -type ResultsList struct { - Node string `json:"node"` - Cid string `json:"cid"` - Folder string `json:"folder"` -} - -// Struct to report from the healthz endpoint -type HealthInfo struct { - DiskFreeSpace FreeSpace `json:"FreeSpace"` -} - -type FreeSpace struct { - TMP MountStatus `json:"tmp"` - ROOT MountStatus `json:"root"` -} - -// Creating structure for DiskStatus -type MountStatus struct { - All uint64 `json:"All"` - Used uint64 `json:"Used"` - Free uint64 `json:"Free"` -} - -// Struct to report for VarZ -type VarZ struct { - // TODO: #241 Fill in with varz to report -} diff --git a/pkg/util/env_var.go b/pkg/util/env_var.go deleted file mode 100644 index beeb7045e1..0000000000 --- a/pkg/util/env_var.go +++ /dev/null @@ -1,26 +0,0 @@ -package util - -import "os" - -type EnvParserFunc[T any] func(string) (T, error) - -func GetEnvAs[T any](envVar string, deflt T, parser EnvParserFunc[T]) T { - v := os.Getenv(envVar) - if v != "" { - r, e := parser(v) - if e == nil { - return r - } - } - - return deflt -} - -func GetEnv(envVar string, deflt string) string { - v := os.Getenv(envVar) - if v != "" { - return v - } - - return deflt -} diff --git a/pkg/util/env_var_test.go b/pkg/util/env_var_test.go deleted file mode 100644 index b606960e17..0000000000 --- a/pkg/util/env_var_test.go +++ /dev/null @@ -1,70 +0,0 @@ -//go:build unit || !integration - -package util_test - -import ( - "strconv" - "testing" - "time" - - "github.com/bacalhau-project/bacalhau/pkg/util" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" -) - -type EnvTestSuite struct { - suite.Suite -} - -func TestEnvTestSuite(t *testing.T) { - suite.Run(t, new(EnvTestSuite)) -} - -func (s *EnvTestSuite) TestBoolEnv() { - v := util.GetEnvAs[bool]("TEST_BOOL", true, strconv.ParseBool) - require.True(s.T(), v) - - v = util.GetEnvAs[bool]("TEST_BOOL", false, strconv.ParseBool) - require.False(s.T(), v) - - s.T().Setenv("TEST_BOOL", "1") - v = util.GetEnvAs[bool]("TEST_BOOL", true, strconv.ParseBool) - require.True(s.T(), v) - - s.T().Setenv("TEST_BOOL", "0") - v = util.GetEnvAs[bool]("TEST_BOOL", true, strconv.ParseBool) - require.False(s.T(), v) -} - -func (s *EnvTestSuite) TestDurationEnv() { - v := util.GetEnvAs[time.Duration]("TEST_DUR", time.Minute, time.ParseDuration) - require.Equal(s.T(), time.Minute, v) - - s.T().Setenv("TEST_DUR", "1h") - v = util.GetEnvAs[time.Duration]("TEST_DUR", time.Minute, time.ParseDuration) - require.Equal(s.T(), time.Hour, v) -} - -func (s *EnvTestSuite) TestIntegerEnv() { - partial := func(base int) func(string) (int64, error) { - return func(k string) (int64, error) { - return strconv.ParseInt(k, base, 0) - } - } - v := util.GetEnvAs[int64]("TEST_INT", 0, partial(10)) - require.Equal(s.T(), int64(0), v) - - s.T().Setenv("TEST_INT", "100") - v = util.GetEnvAs[int64]("TEST_INT", 0, partial(10)) - require.Equal(s.T(), int64(100), v) - -} - -func (s *EnvTestSuite) TestStringEnv() { - v := util.GetEnv("TEST_STR", "hello") - require.Equal(s.T(), "hello", v) - - s.T().Setenv("TEST_STR", "world") - v = util.GetEnv("TEST_STR", "hello") - require.Equal(s.T(), "world", v) -} diff --git a/pkg/util/error_cleanup.go b/pkg/util/error_cleanup.go deleted file mode 100644 index b44d712e3a..0000000000 --- a/pkg/util/error_cleanup.go +++ /dev/null @@ -1,21 +0,0 @@ -package util - -import ( - "context" - "errors" - - "github.com/rs/zerolog/log" -) - -// LogDebugIfContextCanceled will ensure that LOG_LEVEL is set to debug if -// the context is canceled. -func LogDebugIfContextCancelled(ctx context.Context, cleanupErr error, msg string) { - if cleanupErr == nil { - return - } - if !errors.Is(cleanupErr, context.Canceled) { - log.Ctx(ctx).Error().Err(cleanupErr).Msg("failed to close " + msg) - } else { - log.Ctx(ctx).Debug().Err(cleanupErr).Msgf("Context canceled: %s", msg) - } -} diff --git a/pkg/util/filecopy/copy.go b/pkg/util/filecopy/copy.go deleted file mode 100644 index cfa635e1cb..0000000000 --- a/pkg/util/filecopy/copy.go +++ /dev/null @@ -1,82 +0,0 @@ -package filecopy - -import ( - "io" - "io/fs" - "os" - "path" - - "github.com/pkg/errors" -) - -// File copies a single file from src to dst, preserving file mode. -// If the destination file exists, it will be overwritten. -// This may be replaced in future with -// https://github.com/golang/go/issues/62484 -func CopyFile(src, dst string) error { - var err error - - sourceFile, err := os.Open(src) - if err != nil { - return errors.Wrap(err, "failed to open source file") - } - defer sourceFile.Close() - - destinationFile, err := os.Create(dst) - if err != nil { - return errors.Wrap(err, "failed to open target file") - } - defer destinationFile.Close() - - // Efficient copying of bytes from one stream to another - if _, err = io.Copy(destinationFile, sourceFile); err != nil { - return errors.Wrap(err, "failed to copy file to target") - } - - srcinfo, err := os.Stat(src) - if err != nil { - return errors.Wrap(err, "failed to get file mode") - } - - err = os.Chmod(dst, srcinfo.Mode()) - if err != nil { - return errors.Wrap(err, "failed to set file mode") - } - - return nil -} - -// CopyDir copies a whole directory from source to destination -// This may be replaced in future with -// https://github.com/golang/go/issues/62484 -func CopyDir(source string, destination string) error { - sourceInfo, err := os.Stat(source) - if err != nil { - return err - } - - if err := os.MkdirAll(destination, sourceInfo.Mode()); err != nil { - return err - } - - var entries []fs.DirEntry - if entries, err = os.ReadDir(source); err != nil { - return err - } - - for _, entry := range entries { - src := path.Join(source, entry.Name()) - dst := path.Join(destination, entry.Name()) - - if entry.IsDir() { - if err := CopyDir(src, dst); err != nil { - return errors.Wrap(err, "failed to copy directory") - } - } else { - if err := CopyFile(src, dst); err != nil { - return errors.Wrap(err, "failed to copy file") - } - } - } - return nil -} diff --git a/pkg/util/generic/map.go b/pkg/util/generic/map.go deleted file mode 100644 index 1dbd3f7cf6..0000000000 --- a/pkg/util/generic/map.go +++ /dev/null @@ -1,10 +0,0 @@ -package generic - -// Map transforms a slice into a slice of another type -func Map[F any, T any](inputs []F, f func(F) T) []T { - tees := make([]T, 0, len(inputs)) - for _, input := range inputs { - tees = append(tees, f(input)) - } - return tees -} diff --git a/pkg/util/multiaddresses/multiaddresses.go b/pkg/util/multiaddresses/multiaddresses.go deleted file mode 100644 index 425ef423d9..0000000000 --- a/pkg/util/multiaddresses/multiaddresses.go +++ /dev/null @@ -1,32 +0,0 @@ -package multiaddresses - -import ( - "sort" - - "github.com/multiformats/go-multiaddr" - "golang.org/x/exp/slices" -) - -func SortLocalhostFirst(multiAddresses []multiaddr.Multiaddr) []multiaddr.Multiaddr { - multiAddresses = slices.Clone(multiAddresses) - preferLocalhost := func(m multiaddr.Multiaddr) int { - count := 0 - if _, err := m.ValueForProtocol(multiaddr.P_TCP); err == nil { - count++ - } - if ip, err := m.ValueForProtocol(multiaddr.P_IP4); err == nil { - count++ - if ip == "127.0.0.1" { - count++ - } - } else if ip, err := m.ValueForProtocol(multiaddr.P_IP6); err == nil && ip != "::1" { - count++ - } - return count - } - sort.Slice(multiAddresses, func(i, j int) bool { - return preferLocalhost(multiAddresses[i]) > preferLocalhost(multiAddresses[j]) - }) - - return multiAddresses -} diff --git a/pkg/util/string/cross_platform_normalize.go b/pkg/util/string/cross_platform_normalize.go deleted file mode 100644 index 0f0e33a0f9..0000000000 --- a/pkg/util/string/cross_platform_normalize.go +++ /dev/null @@ -1,36 +0,0 @@ -package string - -import ( - "runtime" - "strings" -) - -// Function that normalizes line endings to platform being run on. -// Useful for tests, but possibly useful elsewhere? -func CrossPlatformNormalizeLineEndings(s string) string { - return crossPlatformNormalizeLineEndings(s, runtime.GOOS) -} - -// Internal only function to allow injecting the platform for testing -func crossPlatformNormalizeLineEndings(s string, platform string) string { - // Detect the platform - lineEnding := "\n" - if platform == "windows" { - lineEnding = "\r\n" - } - - // Use go's built-in splitter to split the string into lines - lines := strings.Split(strings.ReplaceAll(s, "\r\n", "\n"), "\n") - - // Trim all whitespace from empty lines - for i, line := range lines { - if len(strings.TrimSpace(line)) == 0 { - lines[i] = "" - } - } - - // Now recombine the lines with the correct line ending - s = strings.Join(lines, lineEnding) - - return s -} diff --git a/pkg/util/string/cross_platform_normalize_test.go b/pkg/util/string/cross_platform_normalize_test.go deleted file mode 100644 index da4f012f1d..0000000000 --- a/pkg/util/string/cross_platform_normalize_test.go +++ /dev/null @@ -1,37 +0,0 @@ -//go:build unit || !integration - -package string - -import "testing" - -func TestUpdateLineEndingsForPlatform(t *testing.T) { - // Test cases - last test case has spaces and line endings (which are stripped in comparison) - cases := []struct { - name, input, platform, want string - }{ - {"Unix to Unix", "Hello\nWorld\n", "unix", "Hello\nWorld\n"}, - {"Windows to Unix", "Hello\r\nWorld\r\n", "unix", "Hello\nWorld\n"}, - {"Mixed to Unix", "Hello\nWorld\r\n", "unix", "Hello\nWorld\n"}, - {"Unix to Windows", "Hello\nWorld\n", "windows", "Hello\r\nWorld\r\n"}, - {"Windows to Windows", "Hello\r\nWorld\r\n", "windows", "Hello\r\nWorld\r\n"}, - {"Mixed to Windows", "Hello\nWorld\r\n", "windows", "Hello\r\nWorld\r\n"}, - {"Blanks and line endings", ` - - -`, "unix", ` - - -`}, - } - - // Run the test cases - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got := crossPlatformNormalizeLineEndings(tc.input, tc.platform) - want := crossPlatformNormalizeLineEndings(tc.want, tc.platform) - if got != want { - t.Errorf("got %q; want %q", got, want) - } - }) - } -} diff --git a/pkg/util/stringutils/cross_platform_normalize.go b/pkg/util/stringutils/cross_platform_normalize.go deleted file mode 100644 index c1f187a304..0000000000 --- a/pkg/util/stringutils/cross_platform_normalize.go +++ /dev/null @@ -1,38 +0,0 @@ -package stringutils - -import ( - "regexp" - "runtime" - "strings" -) - -// Function that normalizes line endings to platform being run on. -// Useful for tests, but possibly useful elsewhere? -func CrossPlatformNormalizeLineEndings(s string) string { - return crossPlatformNormalizeLineEndings(s, runtime.GOOS) -} - -// Internal only function to allow injecting the platform for testing -func crossPlatformNormalizeLineEndings(s string, platform string) string { - // Detect the platform - lineEnding := "\n" - if platform == "windows" { - lineEnding = "\r\n" - } - - regex, _ := regexp.Compile(`\r?\n`) - s = regex.ReplaceAllString(s, "\n") - lines := strings.Split(s, "\n") - - // Trim all whitespace from empty lines - for i, line := range lines { - if len(strings.TrimSpace(line)) == 0 { - lines[i] = "" - } - } - - // Now recombine the lines with the correct line ending - s = strings.Join(lines, lineEnding) - - return s -} diff --git a/pkg/util/stringutils/cross_platform_normalize_test.go b/pkg/util/stringutils/cross_platform_normalize_test.go deleted file mode 100644 index d726357d3a..0000000000 --- a/pkg/util/stringutils/cross_platform_normalize_test.go +++ /dev/null @@ -1,38 +0,0 @@ -//go:build unit || !integration - -package stringutils - -import "testing" - -func TestUpdateLineEndingsForPlatform(t *testing.T) { - // Test cases - last test case has spaces and line endings (which are stripped in comparison) - cases := []struct { - name, input, platform, want string - }{ - {"Unix to Unix", "Hello\nWorld\n", "linux", "Hello\nWorld\n"}, - {"Windows to Unix", "Hello\r\nWorld\r\n", "linux", "Hello\nWorld\n"}, - {"Mixed to Unix", "Hello\nWorld\r\n", "linux", "Hello\nWorld\n"}, - {"Unix to Windows", "Hello\nWorld\n", "windows", "Hello\r\nWorld\r\n"}, - {"Windows to Windows", "Hello\r\nWorld\r\n", "windows", "Hello\r\nWorld\r\n"}, - {"Mixed to Windows", "Hello\nWorld\r\n", "windows", "Hello\r\nWorld\r\n"}, - {"Example string linux", "Create a job from a file or from stdin.\n\n JSON and YAML formats are accepted.", "linux", "Create a job from a file or from stdin.\n\n JSON and YAML formats are accepted."}, - {"Example string windows", "Create a job from a file or from stdin.\r\n\r\n JSON and YAML formats are accepted.", "windows", "Create a job from a file or from stdin.\r\n\r\n JSON and YAML formats are accepted."}, - {"Blanks and line endings", ` - - -`, "unix", ` - - -`}, - } - - // Run the test cases - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got := crossPlatformNormalizeLineEndings(tc.input, tc.platform) - if got != tc.want { - t.Errorf("got %q; want %q", got, tc.want) - } - }) - } -} diff --git a/pkg/util/templates/command_groups.go b/pkg/util/templates/command_groups.go deleted file mode 100644 index 02ee43c672..0000000000 --- a/pkg/util/templates/command_groups.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package templates - -import ( - "github.com/spf13/cobra" -) - -type CommandGroup struct { - Message string - Commands []*cobra.Command -} - -type CommandGroups []CommandGroup - -func (g CommandGroups) Add(c *cobra.Command) { - for _, group := range g { - c.AddCommand(group.Commands...) - } -} - -func (g CommandGroups) Has(c *cobra.Command) bool { - for _, group := range g { - for _, command := range group.Commands { - if command == c { - return true - } - } - } - return false -} - -func AddAdditionalCommands(g CommandGroups, message string, cmds []*cobra.Command) CommandGroups { - group := CommandGroup{Message: message} - for _, c := range cmds { - // Don't show commands that have no short description - if !g.Has(c) && c.Short != "" { - group.Commands = append(group.Commands, c) - } - } - if len(group.Commands) == 0 { - return g - } - return append(g, group) -} diff --git a/pkg/util/templates/help_printer.go b/pkg/util/templates/help_printer.go deleted file mode 100644 index e9144394da..0000000000 --- a/pkg/util/templates/help_printer.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package templates - -import ( - "bytes" - "fmt" - "io" - "strings" - - "github.com/mitchellh/go-wordwrap" - flag "github.com/spf13/pflag" -) - -const offset = 10 - -// HelpFlagPrinter is a printer that -// processes the help flag and print -// it to i/o writer -type HelpFlagPrinter struct { - wrapLimit uint - out io.Writer -} - -// NewHelpFlagPrinter will initialize a HelpFlagPrinter given the -// i/o writer -func NewHelpFlagPrinter(out io.Writer, wrapLimit uint) *HelpFlagPrinter { - return &HelpFlagPrinter{ - wrapLimit: wrapLimit, - out: out, - } -} - -// PrintHelpFlag will beautify the help flags and print it out to p.out -func (p *HelpFlagPrinter) PrintHelpFlag(flag *flag.Flag) { - formatBuf := new(bytes.Buffer) - writeFlag(formatBuf, flag) - - wrappedStr := formatBuf.String() - flagAndUsage := strings.Split(formatBuf.String(), "\n") - flagStr := flagAndUsage[0] - - // if the flag usage is longer than one line, wrap it again - if len(flagAndUsage) > 1 { - nextLines := strings.Join(flagAndUsage[1:], " ") - wrappedUsages := wordwrap.WrapString(nextLines, p.wrapLimit-offset) - wrappedStr = flagStr + "\n" + wrappedUsages - } - appendTabStr := strings.ReplaceAll(wrappedStr, "\n", "\n\t") - - fmt.Fprintf(p.out, "%s", appendTabStr+"\n\n") -} - -// writeFlag will output the help flag based -// on the format provided by getFlagFormat to i/o writer -func writeFlag(out io.Writer, f *flag.Flag) { - deprecated := "" - if f.Deprecated != "" { - deprecated = fmt.Sprintf(" (DEPRECATED: %s)", f.Deprecated) - } - fmt.Fprintf(out, getFlagFormat(f), f.Shorthand, f.Name, f.DefValue, f.Usage, deprecated) -} diff --git a/pkg/util/templates/normalizers.go b/pkg/util/templates/normalizers.go deleted file mode 100644 index 513a7d8fe3..0000000000 --- a/pkg/util/templates/normalizers.go +++ /dev/null @@ -1,15 +0,0 @@ -package templates - -import ( - "k8s.io/kubectl/pkg/util/templates" -) - -// LongDesc normalizes a command's long description to follow the conventions. -func LongDesc(s string) string { - return templates.LongDesc(s) -} - -// Examples normalizes a command's examples to follow the conventions. -func Examples(s string) string { - return templates.Examples(s) -} diff --git a/pkg/util/templates/normalizers_test.go b/pkg/util/templates/normalizers_test.go deleted file mode 100644 index e9ad457ce3..0000000000 --- a/pkg/util/templates/normalizers_test.go +++ /dev/null @@ -1,55 +0,0 @@ -//go:build unit || !integration - -package templates - -import ( - "runtime" - "testing" - - "github.com/bacalhau-project/bacalhau/pkg/util/stringutils" - - "github.com/stretchr/testify/assert" -) - -func TestLongDesc(t *testing.T) { - actual := LongDesc(` - Create a job from a file or from stdin. - - JSON and YAML formats are accepted. - -`) - - actual = stringutils.CrossPlatformNormalizeLineEndings(actual) - want := `Create a job from a file or from stdin. - - JSON and YAML formats are accepted.` - want = stringutils.CrossPlatformNormalizeLineEndings(want) - - assert.Equal(t, want, actual) -} - -func TestExamples(t *testing.T) { - actual := Examples(` # Describe a job with the full ID - bacalhau describe j-e3f8c209-d683-4a41-b840-f09b88d087b9 - - # Describe a job with the a shortened ID - bacalhau describe j-47805f5c - - # Describe a job and include all server and local events - bacalhau describe --include-events j-b6ad164a`) - - actual = stringutils.CrossPlatformNormalizeLineEndings(actual) - - want := ` # Describe a job with the full ID - bacalhau describe j-e3f8c209-d683-4a41-b840-f09b88d087b9 - - # Describe a job with the a shortened ID - bacalhau describe j-47805f5c - - # Describe a job and include all server and local events - bacalhau describe --include-events j-b6ad164a` - - want = stringutils.CrossPlatformNormalizeLineEndings(want) - - assert.Equal(t, want, actual, "Examples did not match - GOOS: %s\nGOARCH: %s", runtime.GOOS, runtime.GOARCH) -} diff --git a/pkg/util/templates/templater.go b/pkg/util/templates/templater.go deleted file mode 100644 index 9be12edbd2..0000000000 --- a/pkg/util/templates/templater.go +++ /dev/null @@ -1,306 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package templates - -import ( - "bytes" - "fmt" - "strings" - "text/template" - "unicode" - - "github.com/spf13/cobra" - flag "github.com/spf13/pflag" - - "k8s.io/kubectl/pkg/util/term" -) - -type FlagExposer interface { - ExposeFlags(cmd *cobra.Command, flags ...string) FlagExposer -} - -func ActsAsRootCommand(cmd *cobra.Command, filters []string, groups ...CommandGroup) FlagExposer { - if cmd == nil { - panic("nil root command") - } - templater := &templater{ - RootCmd: cmd, - UsageTemplate: MainUsageTemplate(), - HelpTemplate: MainHelpTemplate(), - CommandGroups: groups, - Filtered: filters, - } - cmd.SetFlagErrorFunc(templater.FlagErrorFunc()) - cmd.SilenceUsage = true - cmd.SetUsageFunc(templater.UsageFunc()) - cmd.SetHelpFunc(templater.HelpFunc()) - return templater -} - -func UseOptionsTemplates(cmd *cobra.Command) { - templater := &templater{ - UsageTemplate: OptionsUsageTemplate(), - HelpTemplate: OptionsHelpTemplate(), - } - cmd.SetUsageFunc(templater.UsageFunc()) - cmd.SetHelpFunc(templater.HelpFunc()) -} - -type templater struct { - UsageTemplate string - HelpTemplate string - RootCmd *cobra.Command - CommandGroups - Filtered []string -} - -func (t *templater) FlagErrorFunc(exposedFlags ...string) func(*cobra.Command, error) error { - return func(c *cobra.Command, err error) error { - c.SilenceUsage = true - switch c.CalledAs() { - case "options": - return fmt.Errorf("%s\nRun '%s' without flags", err, c.CommandPath()) - default: - return fmt.Errorf("%s\nSee '%s --help' for usage", err, c.CommandPath()) - } - } -} - -func (t *templater) ExposeFlags(cmd *cobra.Command, flags ...string) FlagExposer { - cmd.SetUsageFunc(t.UsageFunc(flags...)) - return t -} - -func (t *templater) HelpFunc() func(*cobra.Command, []string) { - return func(c *cobra.Command, s []string) { - tt := template.New("help") - tt.Funcs(t.templateFuncs()) - template.Must(tt.Parse(t.HelpTemplate)) - out := term.NewResponsiveWriter(c.OutOrStdout()) - err := tt.Execute(out, c) - if err != nil { - c.Println(err) - } - } -} - -func (t *templater) UsageFunc(exposedFlags ...string) func(*cobra.Command) error { - return func(c *cobra.Command) error { - tt := template.New("usage") - tt.Funcs(t.templateFuncs(exposedFlags...)) - template.Must(tt.Parse(t.UsageTemplate)) - out := term.NewResponsiveWriter(c.OutOrStderr()) - return tt.Execute(out, c) - } -} - -func (t *templater) templateFuncs(exposedFlags ...string) template.FuncMap { - return template.FuncMap{ - "trim": strings.TrimSpace, - "trimRight": func(s string) string { return strings.TrimRightFunc(s, unicode.IsSpace) }, - "trimLeft": func(s string) string { return strings.TrimLeftFunc(s, unicode.IsSpace) }, - "gt": cobra.Gt, - "eq": cobra.Eq, - "rpad": rpad, - "appendIfNotPresent": appendIfNotPresent, - "flagsNotIntersected": flagsNotIntersected, - "visibleFlags": visibleFlags, - "flagsUsages": flagsUsages, - "cmdGroups": t.cmdGroups, - "cmdGroupsString": t.cmdGroupsString, - "rootCmd": t.rootCmdName, - "isRootCmd": t.isRootCmd, - "optionsCmdFor": t.optionsCmdFor, - "usageLine": t.usageLine, - "exposed": func(c *cobra.Command) *flag.FlagSet { - exposed := flag.NewFlagSet("exposed", flag.ContinueOnError) - if len(exposedFlags) > 0 { - for _, name := range exposedFlags { - if flag := c.Flags().Lookup(name); flag != nil { - exposed.AddFlag(flag) - } - } - } - return exposed - }, - } -} - -func (t *templater) cmdGroups(c *cobra.Command, all []*cobra.Command) []CommandGroup { - if len(t.CommandGroups) > 0 && c == t.RootCmd { - all = filter(all, t.Filtered...) - return AddAdditionalCommands(t.CommandGroups, "Other Commands:", all) - } - all = filter(all, "options") - return []CommandGroup{ - { - Message: "Available Commands:", - Commands: all, - }, - } -} - -func (t *templater) cmdGroupsString(c *cobra.Command) string { - groups := []string{} - for _, cmdGroup := range t.cmdGroups(c, c.Commands()) { - cmds := []string{cmdGroup.Message} - for _, cmd := range cmdGroup.Commands { - if cmd.IsAvailableCommand() { - cmds = append(cmds, " "+rpad(cmd.Name(), cmd.NamePadding())+" "+cmd.Short) - } - } - groups = append(groups, strings.Join(cmds, "\n")) - } - return strings.Join(groups, "\n\n") -} - -func (t *templater) rootCmdName(c *cobra.Command) string { - return t.rootCmd(c).CommandPath() -} - -func (t *templater) isRootCmd(c *cobra.Command) bool { - return t.rootCmd(c) == c -} - -func (t *templater) parents(c *cobra.Command) []*cobra.Command { - parents := []*cobra.Command{c} - for current := c; !t.isRootCmd(current) && current.HasParent(); { - current = current.Parent() - parents = append(parents, current) - } - return parents -} - -func (t *templater) rootCmd(c *cobra.Command) *cobra.Command { - if c != nil && !c.HasParent() { - return c - } - if t.RootCmd == nil { - panic("nil root cmd") - } - return t.RootCmd -} - -func (t *templater) optionsCmdFor(c *cobra.Command) string { - if !c.Runnable() { - return "" - } - rootCmdStructure := t.parents(c) - for i := len(rootCmdStructure) - 1; i >= 0; i-- { - cmd := rootCmdStructure[i] - if _, _, err := cmd.Find([]string{"options"}); err == nil { - return cmd.CommandPath() + " options" - } - } - return "" -} - -func (t *templater) usageLine(c *cobra.Command) string { - usage := c.UseLine() - suffix := "[options]" - if c.HasFlags() && !strings.Contains(usage, suffix) { - usage += " " + suffix - } - return usage -} - -// flagsUsages will print out the kubectl help flags -func flagsUsages(f *flag.FlagSet) (string, error) { - flagBuf := new(bytes.Buffer) - wrapLimit, err := term.GetWordWrapperLimit() - if err != nil { - wrapLimit = 0 - } - printer := NewHelpFlagPrinter(flagBuf, wrapLimit) - - f.VisitAll(func(flag *flag.Flag) { - if flag.Hidden { - return - } - printer.PrintHelpFlag(flag) - }) - - return flagBuf.String(), nil -} - -// getFlagFormat will output the flag format -func getFlagFormat(f *flag.Flag) string { - var format string - format = "--%s=%s:\n%s%s" - if f.Value.Type() == "string" { - format = "--%s='%s':\n%s%s" - } - - if len(f.Shorthand) > 0 { - format = " -%s, " + format - } else { - format = " %s" + format - } - - return format -} - -func rpad(s string, padding int) string { - template := fmt.Sprintf("%%-%ds", padding) - return fmt.Sprintf(template, s) -} - -func appendIfNotPresent(s, stringToAppend string) string { - if strings.Contains(s, stringToAppend) { - return s - } - return s + " " + stringToAppend -} - -func flagsNotIntersected(l, r *flag.FlagSet) *flag.FlagSet { - f := flag.NewFlagSet("notIntersected", flag.ContinueOnError) - l.VisitAll(func(flag *flag.Flag) { - if r.Lookup(flag.Name) == nil { - f.AddFlag(flag) - } - }) - return f -} - -func visibleFlags(l *flag.FlagSet) *flag.FlagSet { - hidden := "help" - f := flag.NewFlagSet("visible", flag.ContinueOnError) - l.VisitAll(func(flag *flag.Flag) { - if flag.Name != hidden { - f.AddFlag(flag) - } - }) - return f -} - -func filter(cmds []*cobra.Command, names ...string) []*cobra.Command { - out := []*cobra.Command{} - for _, c := range cmds { - if c.Hidden { - continue - } - skip := false - for _, name := range names { - if name == c.Name() { - skip = true - break - } - } - if skip { - continue - } - out = append(out, c) - } - return out -} diff --git a/pkg/util/templates/templates.go b/pkg/util/templates/templates.go deleted file mode 100644 index 0978697f72..0000000000 --- a/pkg/util/templates/templates.go +++ /dev/null @@ -1,96 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package templates - -import ( - "strings" - "unicode" -) - -const ( - // SectionVars is the help template section that declares variables to be used in the template. - SectionVars = `{{$isRootCmd := isRootCmd .}}` + - `{{$rootCmd := rootCmd .}}` + - `{{$visibleFlags := visibleFlags (flagsNotIntersected .LocalFlags .PersistentFlags)}}` + - `{{$explicitlyExposedFlags := exposed .}}` + - `{{$optionsCmdFor := optionsCmdFor .}}` + - `{{$usageLine := usageLine .}}` - - // SectionAliases is the help template section that displays command aliases. - SectionAliases = `{{if gt .Aliases 0}}Aliases: -{{.NameAndAliases}} -{{end}}` - - // SectionExamples is the help template section that displays command examples. - SectionExamples = `{{if .HasExample}}Examples: -{{trimRight .Example}} -{{end}}` - - // SectionSubcommands is the help template section that displays the command's subcommands. - SectionSubcommands = `{{if .HasAvailableSubCommands}}{{cmdGroupsString .}} -{{end}}` - - //nolint:lll // template - // SectionFlags is the help template section that displays the command's flags. - SectionFlags = `{{ if or $visibleFlags.HasFlags $explicitlyExposedFlags.HasFlags}}Options: -{{ if $visibleFlags.HasFlags}}{{trimRight (flagsUsages $visibleFlags)}}{{end}}{{ if $explicitlyExposedFlags.HasFlags}}{{ if $visibleFlags.HasFlags}} -{{end}}{{trimRight (flagsUsages $explicitlyExposedFlags)}}{{end}} -{{end}}` - - // SectionUsage is the help template section that displays the command's usage. - SectionUsage = `{{if and .Runnable (ne .UseLine "") (ne .UseLine $rootCmd)}}Usage: - {{$usageLine}} -{{end}}` - - // SectionTipsHelp is the help template section that displays the '--help' hint. - SectionTipsHelp = `{{if .HasSubCommands}}Use "{{$rootCmd}} --help" for more information about a given command. -{{end}}` - - //nolint:lll // template - // SectionTipsGlobalOptions is the help template section that displays the 'options' hint for displaying global flags. - SectionTipsGlobalOptions = `{{if $optionsCmdFor}}Use "{{$optionsCmdFor}}" for a list of global command-line options (applies to all commands). -{{end}}` -) - -// MainHelpTemplate if the template for 'help' used by most commands. -func MainHelpTemplate() string { - return `{{with or .Long .Short }}{{. | trim}}{{end}}{{if or .Runnable .HasSubCommands}}{{.UsageString}}{{end}}` -} - -// MainUsageTemplate if the template for 'usage' used by most commands. -func MainUsageTemplate() string { - sections := []string{ - "\n\n", - SectionVars, - SectionAliases, - SectionExamples, - SectionSubcommands, - SectionFlags, - SectionUsage, - SectionTipsHelp, - SectionTipsGlobalOptions, - } - return strings.TrimRightFunc(strings.Join(sections, ""), unicode.IsSpace) -} - -// OptionsHelpTemplate if the template for 'help' used by the 'options' command. -func OptionsHelpTemplate() string { - return "" -} - -// OptionsUsageTemplate if the template for 'usage' used by the 'options' command. -func OptionsUsageTemplate() string { - return `{{ if .HasInheritedFlags}}The following options can be passed to any command: -{{flagsUsages .InheritedFlags}}{{end}}` -}