Merge branch 'main' into dependabot/go_modules/github.com/ipld/go-car…

…/v2-2.14.2

.cspell/custom-dictionary.txt

@@ -20,6 +20,7 @@ boltdb
 booga
 boxo
 bprotocol
+nclprotocol
 BRSNW
 BUCKETNAME
 buildx
@@ -438,4 +439,9 @@ buildvcs
 Nilf
 IMDS
 tlsca
-Lenf
+Lenf
+traefik
+bprotocolcompute
+bprotocolorchestrator
+nclprotocolcompute
+ncltest

.github/workflows/build.yml

@@ -30,6 +30,8 @@ jobs:
     steps:
       - name: Install earthly
         uses: earthly/actions-setup@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - uses: actions/checkout@v4
         with:

.github/workflows/lint.yml

@@ -33,6 +33,12 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      # Install SQLite before asdf to ensure Python is built with SQLite support
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libsqlite3-dev
+
       - name: Set Go Version
         shell: bash
         run: |

.golangci.yml

@@ -7,6 +7,9 @@ linters-settings:
             desc: "logging is allowed only by zerolog"
   dupl:
     threshold: 100
+  forbidigo:
+    forbid:
+      - 'fmt\.Print.*'  # This will catch Printf, Println, Print
   funlen:
     lines: 100
     statements: 100
@@ -70,6 +73,7 @@ linters:
   - dogsled
   - errcheck
   - copyloopvar
+  - forbidigo
   - funlen
   - gochecknoinits
   - goconst

buildkite/pipelines/bacalhau-golang.yaml

@@ -54,29 +54,12 @@ steps:
     agents:
       queue: "buildkite-hosted-linux-large"
 
-  - label: ":testengine: Unit Test (NCL Protocol)"
-    command: "./buildkite/scripts/test.sh unit"
-    key: "unit-test-ncl"
-    env:
-      BACALHAU_PREFER_NCL_PROTOCOL: "true"
-    agents:
-      queue: "buildkite-hosted-linux-large"
-
   - label: ":testengine: Integration Test"
     command: "./buildkite/scripts/test.sh integration"
     key: "integration-test"
     agents:
       queue: "buildkite-hosted-linux-large"
 
-
-  - label: ":testengine: Integration Test (NCL Protocol)"
-    command: "./buildkite/scripts/test.sh integration"
-    key: "integration-test-ncl"
-    env:
-      BACALHAU_PREFER_NCL_PROTOCOL: "true"
-    agents:
-      queue: "buildkite-hosted-linux-large"
-
   - wait: ~
 
   - label: ":docker: Build & Publish Bacalhau Image"

cmd/cli/agent/node.go

@@ -56,7 +56,7 @@ func (o *NodeOptions) runNode(cmd *cobra.Command, api client.API) error {
 		return fmt.Errorf("could not get server node: %w", err)
 	}
 
-	writeErr := output.OutputOneNonTabular(cmd, o.OutputOpts, response.NodeState)
+	writeErr := output.OutputOneNonTabular(cmd, o.OutputOpts, response.NodeInfo)
 	if writeErr != nil {
 		return fmt.Errorf("failed to write node: %w", writeErr)
 	}

cmd/cli/agent/node_test.go

@@ -26,18 +26,18 @@ func (s *NodeSuite) TestNodeJSONOutput() {
 	_, out, err := s.ExecuteTestCobraCommand("agent", "node", "--output", string(output.JSONFormat))
 	s.Require().NoError(err, "Could not request node with json output.")
 
-	nodeInfo := &models.NodeState{}
+	nodeInfo := &models.NodeInfo{}
 	err = marshaller.JSONUnmarshalWithMax([]byte(out), &nodeInfo)
 	s.Require().NoError(err, "Could not unmarshal the output into json - %+v", out)
-	s.Require().Equal(s.Node.ID, nodeInfo.Info.ID(), "Node ID does not match in json.")
+	s.Require().Equal(s.Node.ID, nodeInfo.ID(), "Node ID does not match in json.")
 }
 
 func (s *NodeSuite) TestNodeYAMLOutput() {
 	_, out, err := s.ExecuteTestCobraCommand("agent", "node")
 	s.Require().NoError(err, "Could not request node with yaml output.")
 
-	nodeInfo := &models.NodeState{}
+	nodeInfo := &models.NodeInfo{}
 	err = marshaller.YAMLUnmarshalWithMax([]byte(out), &nodeInfo)
 	s.Require().NoError(err, "Could not unmarshal the output into yaml - %+v", out)
-	s.Require().Equal(s.Node.ID, nodeInfo.Info.ID(), "Node ID does not match in yaml.")
+	s.Require().Equal(s.Node.ID, nodeInfo.ID(), "Node ID does not match in yaml.")
 }

cmd/cli/job/executions.go

@@ -106,12 +106,18 @@ var (
 		Value:        func(e *models.Execution) string { return output.Elapsed(e.GetModifyTime()) },
 	}
 	executionColumnID = output.TableColumn[*models.Execution]{
-		ColumnConfig: table.ColumnConfig{Name: "ID", WidthMax: 10, WidthMaxEnforcer: text.WrapText},
-		Value:        func(e *models.Execution) string { return idgen.ShortUUID(e.ID) },
+		ColumnConfig: table.ColumnConfig{
+			Name:             "ID",
+			WidthMax:         idgen.ShortIDLengthWithPrefix,
+			WidthMaxEnforcer: func(col string, maxLen int) string { return idgen.ShortUUID(col) }},
+		Value: func(e *models.Execution) string { return e.ID },
 	}
 	executionColumnNodeID = output.TableColumn[*models.Execution]{
-		ColumnConfig: table.ColumnConfig{Name: "Node ID", WidthMax: 10, WidthMaxEnforcer: text.WrapText},
-		Value:        func(e *models.Execution) string { return idgen.ShortNodeID(e.NodeID) },
+		ColumnConfig: table.ColumnConfig{
+			Name:             "Node ID",
+			WidthMax:         idgen.ShortIDLengthWithPrefix,
+			WidthMaxEnforcer: func(col string, maxLen int) string { return idgen.ShortUUID(col) }},
+		Value: func(e *models.Execution) string { return e.NodeID },
 	}
 	executionColumnRev = output.TableColumn[*models.Execution]{
 		ColumnConfig: table.ColumnConfig{Name: "Rev.", WidthMax: 4, WidthMaxEnforcer: text.WrapText},

cmd/cli/job/history.go

@@ -88,7 +88,7 @@ func NewHistoryCmd() *cobra.Command {
 }
 
 var historyColumns = []output.TableColumn[*models.JobHistory]{
-	cols.HistoryTime,
+	cols.HistoryDateTime,
 	cols.HistoryLevel,
 	cols.HistoryExecID,
 	cols.HistoryTopic,

cmd/cli/node/action.go

@@ -6,6 +6,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/bacalhau-project/bacalhau/cmd/util"
+	"github.com/bacalhau-project/bacalhau/pkg/bacerrors"
 	"github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels"
 	"github.com/bacalhau-project/bacalhau/pkg/publicapi/client/v2"
 )
@@ -56,7 +57,7 @@ func (n *NodeActionCmd) run(cmd *cobra.Command, args []string, api client.API) e
 		Message: n.message,
 	})
 	if err != nil {
-		util.Fatal(cmd, fmt.Errorf("could not %s node %s: %w", n.action, nodeID, err), 1)
+		return bacerrors.Wrap(err, "failed to %s node %s", n.action, nodeID)
 	}
 
 	if response.Success {

cmd/cli/node/action_test.go

@@ -41,14 +41,14 @@ func (s *NodeActionSuite) TestListNodes() {
 	nodeID := cells[0]
 
 	// Try to approve, expect failure
-	_, out, err = s.ExecuteTestCobraCommand(
+	_, _, err = s.ExecuteTestCobraCommand(
 		"node",
 		"approve",
 		nodeID,
 	)
-	s.Require().NoError(err)
-	s.Require().Contains(out, "node already approved")
-	s.Require().Contains(out, nodeID)
+	s.Require().Error(err)
+	s.Require().ErrorContains(err, "already approved")
+	s.Require().ErrorContains(err, nodeID)
 
 	// Now reject the node
 	_, out, err = s.ExecuteTestCobraCommand(
@@ -65,8 +65,9 @@ func (s *NodeActionSuite) TestListNodes() {
 		"reject",
 		nodeID,
 	)
-	s.Require().NoError(err)
-	s.Require().Contains(out, "node already rejected")
+	s.Require().Error(err)
+	s.Require().ErrorContains(err, "already rejected")
+	s.Require().ErrorContains(err, nodeID)
 
 	// Set it to approve again
 	_, out, err = s.ExecuteTestCobraCommand(

cmd/cli/node/columns.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/c2h5oh/datasize"
+	"github.com/dustin/go-humanize"
 	"github.com/jedib0t/go-pretty/v6/table"
 	"github.com/jedib0t/go-pretty/v6/text"
 	"github.com/samber/lo"
@@ -17,8 +17,11 @@ import (
 
 var alwaysColumns = []output.TableColumn[*models.NodeState]{
 	{
-		ColumnConfig: table.ColumnConfig{Name: "id"},
-		Value:        func(node *models.NodeState) string { return idgen.ShortNodeID(node.Info.ID()) },
+		ColumnConfig: table.ColumnConfig{
+			Name:             "id",
+			WidthMax:         idgen.ShortIDLengthWithPrefix,
+			WidthMaxEnforcer: func(col string, maxLen int) string { return idgen.ShortNodeID(col) }},
+		Value: func(node *models.NodeState) string { return node.Info.ID() },
 	},
 	{
 		ColumnConfig: table.ColumnConfig{Name: "type"},
@@ -31,7 +34,7 @@ var alwaysColumns = []output.TableColumn[*models.NodeState]{
 	{
 		ColumnConfig: table.ColumnConfig{Name: "status"},
 		Value: func(ni *models.NodeState) string {
-			return ni.Connection.String()
+			return ni.ConnectionState.Status.String()
 		},
 	},
 }
@@ -70,45 +73,49 @@ var toggleColumns = map[string][]output.TableColumn[*models.NodeState]{
 	"features": {
 		{
 			ColumnConfig: table.ColumnConfig{Name: "engines", WidthMax: maxLen(models.EngineNames), WidthMaxEnforcer: text.WrapSoft},
-			Value: ifComputeNode(func(cni *models.ComputeNodeInfo) string {
+			Value: ifComputeNode(func(cni models.ComputeNodeInfo) string {
 				return strings.Join(cni.ExecutionEngines, " ")
 			}),
 		},
 		{
 			ColumnConfig: table.ColumnConfig{Name: "inputs from", WidthMax: maxLen(models.StoragesNames), WidthMaxEnforcer: text.WrapSoft},
-			Value: ifComputeNode(func(cni *models.ComputeNodeInfo) string {
+			Value: ifComputeNode(func(cni models.ComputeNodeInfo) string {
 				return strings.Join(cni.StorageSources, " ")
 			}),
 		},
 		{
 			ColumnConfig: table.ColumnConfig{Name: "outputs", WidthMax: maxLen(models.PublisherNames), WidthMaxEnforcer: text.WrapSoft},
-			Value: ifComputeNode(func(cni *models.ComputeNodeInfo) string {
+			Value: ifComputeNode(func(cni models.ComputeNodeInfo) string {
 				return strings.Join(cni.Publishers, " ")
 			}),
 		},
 	},
 	"capacity": {
 		{
 			ColumnConfig: table.ColumnConfig{Name: "cpu", WidthMax: len("1.0 / "), WidthMaxEnforcer: text.WrapSoft},
-			Value: ifComputeNode(func(cni *models.ComputeNodeInfo) string {
+			Value: ifComputeNode(func(cni models.ComputeNodeInfo) string {
 				return fmt.Sprintf("%.1f / %.1f", cni.AvailableCapacity.CPU, cni.MaxCapacity.CPU)
 			}),
 		},
 		{
 			ColumnConfig: table.ColumnConfig{Name: "memory", WidthMax: len("10.0 GB / "), WidthMaxEnforcer: text.WrapSoft},
-			Value: ifComputeNode(func(cni *models.ComputeNodeInfo) string {
-				return fmt.Sprintf("%s / %s", datasize.ByteSize(cni.AvailableCapacity.Memory).HR(), datasize.ByteSize(cni.MaxCapacity.Memory).HR())
+			Value: ifComputeNode(func(cni models.ComputeNodeInfo) string {
+				return fmt.Sprintf("%s / %s",
+					humanize.Bytes(cni.AvailableCapacity.Memory),
+					humanize.Bytes(cni.MaxCapacity.Memory))
 			}),
 		},
 		{
 			ColumnConfig: table.ColumnConfig{Name: "disk", WidthMax: len("100.0 GB / "), WidthMaxEnforcer: text.WrapSoft},
-			Value: ifComputeNode(func(cni *models.ComputeNodeInfo) string {
-				return fmt.Sprintf("%s / %s", datasize.ByteSize(cni.AvailableCapacity.Disk).HR(), datasize.ByteSize(cni.MaxCapacity.Disk).HR())
+			Value: ifComputeNode(func(cni models.ComputeNodeInfo) string {
+				return fmt.Sprintf("%s / %s",
+					humanize.Bytes(cni.AvailableCapacity.Disk),
+					humanize.Bytes(cni.MaxCapacity.Disk))
 			}),
 		},
 		{
 			ColumnConfig: table.ColumnConfig{Name: "gpu", WidthMax: len("1 / "), WidthMaxEnforcer: text.WrapSoft},
-			Value: ifComputeNode(func(cni *models.ComputeNodeInfo) string {
+			Value: ifComputeNode(func(cni models.ComputeNodeInfo) string {
 				return fmt.Sprintf("%d / %d", cni.AvailableCapacity.GPU, cni.MaxCapacity.GPU)
 			}),
 		},
@@ -119,9 +126,9 @@ func maxLen(val []string) int {
 	return lo.Max(lo.Map[string, int](val, func(item string, index int) int { return len(item) })) + 1
 }
 
-func ifComputeNode(getFromCNInfo func(*models.ComputeNodeInfo) string) func(state *models.NodeState) string {
+func ifComputeNode(getFromCNInfo func(models.ComputeNodeInfo) string) func(state *models.NodeState) string {
 	return func(ni *models.NodeState) string {
-		if ni.Info.ComputeNodeInfo == nil {
+		if !ni.Info.IsComputeNode() {
 			return ""
 		}
 		return getFromCNInfo(ni.Info.ComputeNodeInfo)

cmd/cli/serve/serve.go

@@ -229,42 +229,46 @@ func serve(cmd *cobra.Command, cfg types.Bacalhau, fsRepo *repo.FsRepo) error {
 		}()
 	}
 
+	isDebug := system.IsDebugMode()
 	startupLog := log.Info().
-		Str("name", sysmeta.NodeName).
-		Str("address", fmt.Sprintf("%s:%d", hostAddress, cfg.API.Port)).
-		Bool("compute_enabled", cfg.Compute.Enabled).
-		Bool("orchestrator_enabled", cfg.Orchestrator.Enabled).
-		Bool("webui_enabled", cfg.WebUI.Enabled)
-	if cfg.Compute.Enabled {
-		capacity := standardNode.ComputeNode.Capacity.GetMaxCapacity(ctx)
+		Str("name", sysmeta.NodeName)
+
+	if isDebug {
 		startupLog.
-			Strs("engines", standardNode.ComputeNode.Executors.Keys(ctx)).
-			Strs("publishers", standardNode.ComputeNode.Publishers.Keys(ctx)).
-			Strs("storages", standardNode.ComputeNode.Storages.Keys(ctx)).
-			Strs("orchestrators", cfg.Compute.Orchestrators).
-			Str("capacity", capacity.String())
-
-		if len(cfg.Compute.AllowListedLocalPaths) > 0 {
-			startupLog.Strs("volumes", cfg.Compute.AllowListedLocalPaths)
+			Str("address", fmt.Sprintf("%s:%d", hostAddress, cfg.API.Port)).
+			Bool("compute_enabled", cfg.Compute.Enabled).
+			Bool("orchestrator_enabled", cfg.Orchestrator.Enabled).
+			Bool("webui_enabled", cfg.WebUI.Enabled)
+	}
+	if cfg.Compute.Enabled {
+		startupLog.Strs("orchestrators", cfg.Compute.Orchestrators)
+
+		if isDebug {
+			capacity := standardNode.ComputeNode.Capacity.GetMaxCapacity(ctx)
+			startupLog.
+				Strs("engines", standardNode.ComputeNode.Executors.Keys(ctx)).
+				Strs("publishers", standardNode.ComputeNode.Publishers.Keys(ctx)).
+				Strs("storages", standardNode.ComputeNode.Storages.Keys(ctx)).
+				Str("capacity", capacity.String())
+			if len(cfg.Compute.AllowListedLocalPaths) > 0 {
+				startupLog.Strs("volumes", cfg.Compute.AllowListedLocalPaths)
+			}
 		}
 	}
 
 	if cfg.Orchestrator.Enabled {
-		startupLog.Str("orchestrator_address",
-			fmt.Sprintf("%s:%d", cfg.Orchestrator.Host, cfg.Orchestrator.Port))
+		if isDebug {
+			startupLog.Str("orchestrator_address",
+				fmt.Sprintf("%s:%d", cfg.Orchestrator.Host, cfg.Orchestrator.Port))
+		}
 	}
 	startupLog.Msg("bacalhau node running")
 
 	envvars := buildEnvVariables(cfg)
-	cmd.Println()
-	cmd.Println("To connect to this node from the local client, run the following commands in your shell:")
-	cmd.Println(envvars)
-
 	riPath, err := fsRepo.WriteRunInfo(ctx, envvars)
 	if err != nil {
 		return err
 	}
-	cmd.Printf("A copy of these variables have been written to: %s\n", riPath)
 	defer os.Remove(riPath)
 
 	<-ctx.Done() // block until killed