feat: adds agent config command that returns the agent configuration (

#4671) - closes #4113 Example: ```shell $ bacalhau agent config ``` ```yaml API: Host: 0.0.0.0 Port: 1234 Auth: Methods: ClientKey: Type: challenge NameProvider: puuid DataDir: /home/frrist/.bacalhau Orchestrator: Enabled: true Host: 0.0.0.0 Port: 4222 NodeManager: DisconnectTimeout: 1m0s Scheduler: WorkerCount: 32 QueueBackoff: 1m0s HousekeepingInterval: 30s HousekeepingTimeout: 2m0s EvaluationBroker: VisibilityTimeout: 1m0s MaxRetryCount: 10 Compute: Orchestrators: - nats://127.0.0.1:4222 Heartbeat: InfoUpdateInterval: 1m0s ResourceUpdateInterval: 30s Interval: 15s AllocatedCapacity: CPU: 70% Memory: 70% Disk: 70% GPU: 100% AllowListedLocalPaths: [] WebUI: Listen: 0.0.0.0:8438 InputSources: ReadTimeout: 5m0s MaxRetryCount: 3 Publishers: Types: Local: Address: 127.0.0.1 Port: 6001 Engines: Types: Docker: ManifestCache: Size: 1000 TTL: 1h0m0s Refresh: 1h0m0s JobDefaults: Batch: Task: Resources: CPU: 500m Memory: 1Gb Ops: Task: Resources: CPU: 500m Memory: 1Gb Daemon: Task: Resources: CPU: 500m Memory: 1Gb Service: Task: Resources: CPU: 500m Memory: 1Gb JobAdmissionControl: Locality: anywhere Logging: Level: info Mode: default LogDebugInfoInterval: 30s UpdateConfig: Interval: 24h0m0s ```  ## Summary by CodeRabbit - **New Features** - Introduced a command to retrieve the agent's configuration, allowing users to access configuration details easily. - Added support for customizable output formats (JSON and YAML) for the configuration command. - **Bug Fixes** - Enhanced error handling and sensitive data redaction in the configuration retrieval process. - **Tests** - Implemented tests to ensure the correct functionality and output of the new configuration command, including validation of sensitive data redaction.  --------- Co-authored-by: frrist <[email protected]>
bacalhau-project · Oct 31, 2024 · 111f238 · 111f238
1 parent 0db0ff2
commit 111f238
Show file tree

Hide file tree

Showing 14 changed files with 201 additions and 21 deletions.
diff --git a/cmd/cli/agent/config.go b/cmd/cli/agent/config.go
@@ -0,0 +1,48 @@
+package agent
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/bacalhau-project/bacalhau/cmd/util"
+	"github.com/bacalhau-project/bacalhau/cmd/util/flags/cliflags"
+	"github.com/bacalhau-project/bacalhau/cmd/util/output"
+	"github.com/bacalhau-project/bacalhau/pkg/config/types"
+	"github.com/bacalhau-project/bacalhau/pkg/publicapi/client/v2"
+)
+
+func NewConfigCmd() *cobra.Command {
+	o := output.NonTabularOutputOptions{
+		Format: output.YAMLFormat,
+		Pretty: true,
+	}
+	configCmd := &cobra.Command{
+		Use:   "config",
+		Short: "Get the agent's configuration.",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := util.SetupRepoConfig(cmd)
+			if err != nil {
+				return fmt.Errorf("failed to setup repo: %w", err)
+			}
+			api, err := util.GetAPIClientV2(cmd, cfg)
+			if err != nil {
+				return fmt.Errorf("failed to create api client: %w", err)
+			}
+			return run(cmd, api, o)
+		},
+	}
+	configCmd.Flags().AddFlagSet(cliflags.OutputNonTabularFormatFlags(&o))
+	return configCmd
+}
+
+func run(cmd *cobra.Command, api client.API, o output.NonTabularOutputOptions) error {
+	ctx := cmd.Context()
+	response, err := api.Agent().Config(ctx)
+	if err != nil {
+		return fmt.Errorf("cannot get agent config: %w", err)
+	}
+
+	return output.OutputOneNonTabular[types.Bacalhau](cmd, o, response.Config)
+}
diff --git a/cmd/cli/agent/config_test.go b/cmd/cli/agent/config_test.go
@@ -0,0 +1,46 @@
+//go:build unit || !integration
+
+package agent_test
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"gopkg.in/yaml.v3"
+
+	cmdtesting "github.com/bacalhau-project/bacalhau/cmd/testing"
+	"github.com/bacalhau-project/bacalhau/cmd/util/output"
+	"github.com/bacalhau-project/bacalhau/pkg/config/types"
+)
+
+func TestConfigSuite(t *testing.T) {
+	suite.Run(t, new(ConfigSuite))
+}
+
+type ConfigSuite struct {
+	cmdtesting.BaseSuite
+}
+
+func (s *ConfigSuite) TestConfigJSONOutput() {
+	_, out, err := s.ExecuteTestCobraCommand(
+		"agent", "config", "--output", string(output.JSONFormat), "--pretty=false",
+	)
+	s.Require().NoError(err, "Could not request config with json output.")
+
+	var cfg types.Bacalhau
+	err = json.Unmarshal([]byte(out), &cfg)
+	s.Require().NoError(err, "Could not unmarshal the output into json - %+v", err)
+	s.Require().True(cfg.Orchestrator.Enabled)
+}
+
+func (s *ConfigSuite) TestConfigYAMLOutput() {
+	// NB: the default output is yaml, thus we don't specify it here.
+	_, out, err := s.ExecuteTestCobraCommand("agent", "config")
+	s.Require().NoError(err, "Could not request config with yaml output.")
+
+	var cfg types.Bacalhau
+	err = yaml.Unmarshal([]byte(out), &cfg)
+	s.Require().NoError(err, "Could not unmarshal the output into yaml - %+v", out)
+	s.Require().True(cfg.Orchestrator.Enabled)
+}
diff --git a/cmd/cli/agent/root.go b/cmd/cli/agent/root.go
@@ -16,5 +16,6 @@ func NewCmd() *cobra.Command {
 	cmd.AddCommand(NewAliveCmd())
 	cmd.AddCommand(NewNodeCmd())
 	cmd.AddCommand(NewVersionCmd())
+	cmd.AddCommand(NewConfigCmd())
 	return cmd
 }
diff --git a/pkg/config/types/bacalhau.go b/pkg/config/types/bacalhau.go
@@ -34,7 +34,8 @@ type Bacalhau struct {
 	Logging             Logging             `yaml:"Logging,omitempty" json:"Logging,omitempty"`
 	UpdateConfig        UpdateConfig        `yaml:"UpdateConfig,omitempty" json:"UpdateConfig,omitempty"`
 	FeatureFlags        FeatureFlags        `yaml:"FeatureFlags,omitempty" json:"FeatureFlags,omitempty"`
-	DisableAnalytics    bool                `yaml:"DisableAnalytics,omitempty" json:"DisableAnalytics,omitempty"`
+	// DisableAnalytics, when true, disables sharing anonymous analytics data with the Bacalhau development team
+	DisableAnalytics bool `yaml:"DisableAnalytics,omitempty" json:"DisableAnalytics,omitempty"`
 }
 
 // Copy returns a deep copy of the Bacalhau configuration.

diff --git a/pkg/publicapi/apimodels/agent.go b/pkg/publicapi/apimodels/agent.go
@@ -1,6 +1,9 @@
 package apimodels
 
-import "github.com/bacalhau-project/bacalhau/pkg/models"
+import (
+	"github.com/bacalhau-project/bacalhau/pkg/config/types"
+	"github.com/bacalhau-project/bacalhau/pkg/models"
+)
 
 // IsAliveResponse is the response to the IsAlive request.
 type IsAliveResponse struct {
@@ -30,3 +33,8 @@ type GetAgentNodeResponse struct {
 	BaseGetResponse
 	*models.NodeState
 }
+
+type GetAgentConfigResponse struct {
+	BaseGetResponse
+	Config types.Bacalhau `json:"config"`
+}
diff --git a/pkg/publicapi/client/v2/api_agent.go b/pkg/publicapi/client/v2/api_agent.go
@@ -30,3 +30,9 @@ func (c *Agent) Node(ctx context.Context, req *apimodels.GetAgentNodeRequest) (*
 	err := c.client.Get(ctx, "/api/v1/agent/node", req, &res)
 	return &res, err
 }
+
+func (c *Agent) Config(ctx context.Context) (*apimodels.GetAgentConfigResponse, error) {
+	var res apimodels.GetAgentConfigResponse
+	err := c.client.Get(ctx, "/api/v1/agent/config", &apimodels.BaseGetRequest{}, &res)
+	return &res, err
+}
diff --git a/pkg/publicapi/endpoint/agent/endpoint.go b/pkg/publicapi/endpoint/agent/endpoint.go
@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/labstack/echo/v4"
@@ -113,7 +114,7 @@ func (e *Endpoint) debug(c echo.Context) error {
 	return c.JSON(http.StatusOK, debugInfoMap)
 }
 
-// debug godoc
+// config godoc
 //
 //	@ID			agent/config
 //	@Summary	Returns the current configuration of the node.
@@ -123,6 +124,17 @@ func (e *Endpoint) debug(c echo.Context) error {
 //	@Failure	500	{object}	string
 //	@Router		/api/v1/agent/config [get]
 func (e *Endpoint) config(c echo.Context) error {
-	cfg := e.bacalhauConfig
-	return c.JSON(http.StatusOK, cfg)
+	cfg, err := e.bacalhauConfig.Copy()
+	if err != nil {
+		return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("could not copy bacalhau config: %s", err))
+	}
+	if cfg.Compute.Auth.Token != "" {
+		cfg.Compute.Auth.Token = "<redacted>"
+	}
+	if cfg.Orchestrator.Auth.Token != "" {
+		cfg.Orchestrator.Auth.Token = "<redacted>"
+	}
+	return c.JSON(http.StatusOK, apimodels.GetAgentConfigResponse{
+		Config: cfg,
+	})
 }
diff --git a/pkg/publicapi/endpoint/agent/endpoint_test.go b/pkg/publicapi/endpoint/agent/endpoint_test.go
@@ -0,0 +1,52 @@
+//go:build unit || !integration
+
+package agent
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/labstack/echo/v4"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/bacalhau-project/bacalhau/pkg/config/types"
+	"github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels"
+)
+
+// TestEndpointConfigRedactFields asserts that auth tokens in the config are redacted.
+func TestEndpointConfigRedactFields(t *testing.T) {
+	router := echo.New()
+
+	// populate the fields that should be redacted with "secret" values.
+	NewEndpoint(EndpointParams{
+		Router: router,
+		BacalhauConfig: types.Bacalhau{
+			Orchestrator: types.Orchestrator{
+				Auth: types.OrchestratorAuth{
+					Token: "super-secret-orchestrator-token",
+				},
+			},
+			Compute: types.Compute{
+				Auth: types.ComputeAuth{
+					Token: "super-secret-orchestrator-token",
+				},
+			},
+		},
+	})
+
+	req := httptest.NewRequest(http.MethodGet, "/api/v1/agent/config", nil)
+	rr := httptest.NewRecorder()
+	router.ServeHTTP(rr, req)
+
+	require.Equal(t, http.StatusOK, rr.Code)
+
+	// assert the secret values are not present.
+	var payload apimodels.GetAgentConfigResponse
+	err := json.NewDecoder(rr.Body).Decode(&payload)
+	require.NoError(t, err)
+	assert.Equal(t, payload.Config.Orchestrator.Auth.Token, "<redacted>")
+	assert.Equal(t, payload.Config.Compute.Auth.Token, "<redacted>")
+}
diff --git a/test_integration/1_orchestrator_basic_config_suite_test.go b/test_integration/1_orchestrator_basic_config_suite_test.go
@@ -3,11 +3,12 @@ package test_integration
 import (
 	"context"
 	"fmt"
+	"strings"
+	"testing"
+
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/suite"
 	"github.com/testcontainers/testcontainers-go/exec"
-	"strings"
-	"testing"
 )
 
 type OrchestratorBasicConfigSuite struct {
@@ -65,7 +66,7 @@ func (s *OrchestratorBasicConfigSuite) TestOrchestratorNodeUpAndEnabled() {
 	marshalledOutput, err := s.unmarshalJSONString(agentConfigOutput, JSONObject)
 	s.Require().NoErrorf(err, "Error unmarshalling response: %q", err)
 
-	orchestratorEnabled := marshalledOutput.(map[string]interface{})["Orchestrator"].(map[string]interface{})["Enabled"].(bool)
+	orchestratorEnabled := marshalledOutput.(map[string]interface{})["config"].(map[string]interface{})["Orchestrator"].(map[string]interface{})["Enabled"].(bool)
 	s.Require().Truef(orchestratorEnabled, "Expected orchestrator to be enabled, got: %t", orchestratorEnabled)
 }
 

diff --git a/test_integration/2_orchestrator_config_override_suite_test.go b/test_integration/2_orchestrator_config_override_suite_test.go
@@ -3,10 +3,11 @@ package test_integration
 import (
 	"context"
 	"fmt"
-	"github.com/google/uuid"
-	"github.com/stretchr/testify/suite"
 	"strings"
 	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/suite"
 )
 
 type OrchestratorConfigOverrideSuite struct {

diff --git a/test_integration/3_orchestrator_config_override_and_flag_suite_test.go b/test_integration/3_orchestrator_config_override_and_flag_suite_test.go
@@ -3,10 +3,11 @@ package test_integration
 import (
 	"context"
 	"fmt"
-	"github.com/google/uuid"
-	"github.com/stretchr/testify/suite"
 	"strings"
 	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/suite"
 )
 
 type OrchestratorConfigOverrideAndFlagSuite struct {

diff --git a/test_integration/4_orchestrator_config_override_and_flag_and_config_flag_suite_test.go b/test_integration/4_orchestrator_config_override_and_flag_and_config_flag_suite_test.go
@@ -3,10 +3,11 @@ package test_integration
 import (
 	"context"
 	"fmt"
-	"github.com/google/uuid"
-	"github.com/stretchr/testify/suite"
 	"strings"
 	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/suite"
 )
 
 type OrchestratorConfigOverrideAndFlagAndConfigFlagSuite struct {
@@ -67,7 +68,7 @@ func (s *OrchestratorConfigOverrideAndFlagAndConfigFlagSuite) TestConfigOverride
 	unmarshalledAgentOutput, err := s.unmarshalJSONString(agentConfigOutput, JSONObject)
 	s.Require().NoErrorf(err, "Error unmarshalling response: %q", err)
 
-	webuiEnabled := unmarshalledAgentOutput.(map[string]interface{})["WebUI"].(map[string]interface{})["Enabled"].(bool)
+	webuiEnabled := unmarshalledAgentOutput.(map[string]interface{})["config"].(map[string]interface{})["WebUI"].(map[string]interface{})["Enabled"].(bool)
 	s.Require().Truef(webuiEnabled, "Expected orchestrator to be enabled, got: %t", webuiEnabled)
 }
 

diff --git a/test_integration/5_orchestrator_no_config_suite_test.go b/test_integration/5_orchestrator_no_config_suite_test.go
@@ -2,10 +2,11 @@ package test_integration
 
 import (
 	"context"
-	"github.com/google/uuid"
-	"github.com/stretchr/testify/suite"
 	"strings"
 	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/suite"
 )
 
 type OrchestratorNoConfigSuite struct {
@@ -56,7 +57,7 @@ func (s *OrchestratorNoConfigSuite) TestStartingOrchestratorNodeWithConfigFile()
 	unmarshalledOutput, err := s.unmarshalJSONString(agentConfigOutput, JSONObject)
 	s.Require().NoErrorf(err, "Error unmarshalling response: %q", err)
 
-	unmarshalledOutputMap := unmarshalledOutput.(map[string]interface{})
+	unmarshalledOutputMap := unmarshalledOutput.(map[string]interface{})["config"].(map[string]interface{})
 
 	orchestratorEnabled := unmarshalledOutputMap["Orchestrator"].(map[string]interface{})["Enabled"].(bool)
 	s.Require().Truef(orchestratorEnabled, "Expected orchestrator to be enabled, got: %t", orchestratorEnabled)

diff --git a/test_integration/6_jobs_basic_runs_scenarios_suite_test.go b/test_integration/6_jobs_basic_runs_scenarios_suite_test.go
@@ -1,14 +1,15 @@
 package test_integration
 
 import (
-	"bacalhau/integration_tests/utils"
 	"context"
 	"fmt"
-	"github.com/google/uuid"
-	"github.com/stretchr/testify/suite"
 	"strings"
 	"testing"
 	"time"
+
+	"bacalhau/integration_tests/utils"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/suite"
 )
 
 type JobsBasicRunsScenariosSuite struct {