Skip to content

Commit

Permalink
debugged deployment
Browse files Browse the repository at this point in the history
  • Loading branch information
@aronchick
aronchick committed Dec 3, 2024
1 parent ab281a2 commit 74ec3ed
Show file tree
Hide file tree
Showing 18 changed files with 604 additions and 645 deletions.
1 change: 1 addition & 0 deletions .cspell/custom-dictionary.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,7 @@ panicnil
PBIP
pdone
pflag
pkill
polandcentral
Pollerer
practise
Expand Down
56 changes: 35 additions & 21 deletions cmd/beta/provision/provisioner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ const (
MaxRetries = 5
RetryDelay = 10 * time.Second
marginSpaces = " "
SSHRetryCount = 10 // Number of SSH connection retries
)

// Provisioner handles the node provisioning process
Expand All @@ -49,7 +50,7 @@ func NewProvisioner(config *NodeConfig) (*Provisioner, error) {
config.IPAddress,
DefaultSSHPort,
config.Username,
config.PrivateKey,
config.PrivateKeyPath,
)
if err != nil {
return nil, fmt.Errorf("failed to create SSH config: %w", err)
Expand All @@ -76,7 +77,7 @@ func validateNodeConfig(config *NodeConfig) error {
if config.Username == "" {
return fmt.Errorf("username is required")
}
if config.PrivateKey == "" {
if config.PrivateKeyPath == "" {
return fmt.Errorf("private key is required")
}
return nil
Expand All @@ -86,7 +87,7 @@ func validateNodeConfig(config *NodeConfig) error {
func createMachineInstance(config *NodeConfig) (models.Machiner, error) {
machine := models.Machine{}
machine.SetSSHUser(config.Username)
machine.SetSSHPrivateKeyPath(config.PrivateKey)
machine.SetSSHPrivateKeyPath(config.PrivateKeyPath)
machine.SetSSHPort(DefaultSSHPort)
machine.SetPublicIP(config.IPAddress)
machine.SetOrchestratorIP("")
Expand Down Expand Up @@ -137,7 +138,7 @@ func (p *Provisioner) ProvisionWithCallback(
StatusMessage: stepRegistry.GetStep(common_interface.SSHConnection).RenderStartMessage(),
})

if err := p.SSHConfig.WaitForSSH(ctx, 3, SSHTimeOut); err != nil { //nolint:mnd
if err := p.SSHConfig.WaitForSSH(ctx, SSHRetryCount, SSHTimeOut); err != nil { //nolint:mnd
progress.CurrentStep.Status = "Failed"
progress.CurrentStep.Error = err
errMsg := fmt.Sprintf("❌ SSH connection failed: %v", err)
Expand Down Expand Up @@ -208,8 +209,13 @@ func (p *Provisioner) ProvisionWithCallback(
// handleProvisionError processes and formats provisioning errors
func handleProvisionError(err error, config *NodeConfig, l *logger.Logger) error {
var cmdOutput string
if sshErr, ok := err.(*sshutils.SSHError); ok {
cmdOutput = sshErr.Output

type outputError interface {
Output() string
}

if outputErr, ok := err.(outputError); ok {
cmdOutput = outputErr.Output()
}

l.Errorf("Provisioning failed with error: %v", err)
Expand All @@ -220,19 +226,18 @@ func handleProvisionError(err error, config *NodeConfig, l *logger.Logger) error
l.Debugf("Full error context:\nIP: %s\nUser: %s\nPrivate Key Path: %s\nError: %v",
config.IPAddress,
config.Username,
config.PrivateKey,
config.PrivateKeyPath,
err)

if cmdOutput != "" {
return fmt.Errorf(
"failed to provision Bacalhau node:\nIP: %s\nCommand Output: %s\nError Details: %w",
config.IPAddress,
cmdOutput,
err,
)
err)
}
return fmt.Errorf("failed to provision Bacalhau node:\nIP: %s\nError Details: %w",
config.IPAddress, err)

return err
}

// GetMachine returns the configured machine instance
Expand Down Expand Up @@ -268,17 +273,26 @@ func (p *Provisioner) ParseSettings(filePath string) ([]models.BacalhauSettings,
// testMode is used to run the provisioner in test mode
var testMode bool

// ProvisionCmd represents the provision command
var ProvisionCmd = &cobra.Command{
Use: "provision",
Short: "Provision a new node",
RunE: runProvision,
}

func runProvision(cmd *cobra.Command, args []string) error {
// Get configuration from flags
config := &NodeConfig{
IPAddress: cmd.Flag("ip").Value.String(),
Username: cmd.Flag("user").Value.String(),
PrivateKeyPath: cmd.Flag("key").Value.String(),
OrchestratorIP: cmd.Flag("orchestrator").Value.String(),
BacalhauSettingsPath: cmd.Flag("bacalhau-settings").Value.String(),
}

cmd.Flags().BoolVar(&testMode, "test", false,
"Run in test mode (simulation only)")

// Validate configuration
if err := config.Validate(); err != nil {
return fmt.Errorf("invalid configuration: %w", err)
if allErrs := config.Validate(); len(allErrs) > 0 {
fmt.Println("Invalid configuration:")
for _, err := range allErrs {
fmt.Println(err)
}
return fmt.Errorf("invalid configuration")
}

// Create new provisioner
Expand All @@ -302,7 +316,7 @@ func runProvision(cmd *cobra.Command, args []string) error {
`,
config.IPAddress,
config.Username,
config.PrivateKey,
config.PrivateKeyPath,
)

// Create a channel for progress updates
Expand Down
36 changes: 18 additions & 18 deletions cmd/beta/provision/provisioner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,9 @@ func (cbpts *CmdBetaProvisionTestSuite) TestNewProvisioner() {
{
name: "valid config",
config: &provision.NodeConfig{
IPAddress: "192.168.1.1",
Username: "testuser",
PrivateKey: cbpts.testSSHPrivateKeyPath,
IPAddress: "192.168.1.1",
Username: "testuser",
PrivateKeyPath: cbpts.testSSHPrivateKeyPath,
},
expectError: false,
},
Expand All @@ -142,17 +142,17 @@ func (cbpts *CmdBetaProvisionTestSuite) TestNewProvisioner() {
{
name: "missing IP",
config: &provision.NodeConfig{
Username: "testuser",
PrivateKey: cbpts.testSSHPrivateKeyPath,
Username: "testuser",
PrivateKeyPath: cbpts.testSSHPrivateKeyPath,
},
expectError: true,
errorMsg: "IP address is required",
},
{
name: "missing username",
config: &provision.NodeConfig{
IPAddress: "192.168.1.1",
PrivateKey: cbpts.testSSHPrivateKeyPath,
IPAddress: "192.168.1.1",
PrivateKeyPath: cbpts.testSSHPrivateKeyPath,
},
expectError: true,
errorMsg: "username is required",
Expand Down Expand Up @@ -184,9 +184,9 @@ func (cbpts *CmdBetaProvisionTestSuite) TestNewProvisioner() {

func (cbpts *CmdBetaProvisionTestSuite) TestProvision() {
config := &provision.NodeConfig{
IPAddress: "192.168.1.1",
Username: "testuser",
PrivateKey: cbpts.testSSHPrivateKeyPath,
IPAddress: "192.168.1.1",
Username: "testuser",
PrivateKeyPath: cbpts.testSSHPrivateKeyPath,
}

originalCalls := cbpts.mockSSHConfig.ExpectedCalls
Expand Down Expand Up @@ -230,7 +230,7 @@ setting.two: "value2"
config := &provision.NodeConfig{
IPAddress: "192.168.1.1",
Username: "testuser",
PrivateKey: cbpts.testSSHPrivateKeyPath,
PrivateKeyPath: cbpts.testSSHPrivateKeyPath,
BacalhauSettingsPath: settingsFile,
}

Expand Down Expand Up @@ -274,7 +274,7 @@ invalid-setting
config := &provision.NodeConfig{
IPAddress: "192.168.1.1",
Username: "testuser",
PrivateKey: cbpts.testSSHPrivateKeyPath,
PrivateKeyPath: cbpts.testSSHPrivateKeyPath,
BacalhauSettingsPath: settingsFile,
}

Expand Down Expand Up @@ -319,9 +319,9 @@ func (cbpts *CmdBetaProvisionTestSuite) TestProvisionWithDockerCheck() {
defer func() { sshutils.NewSSHConfigFunc = origNewSSHConfigFunc }()

config := &provision.NodeConfig{
IPAddress: "192.168.1.1",
Username: "testuser",
PrivateKey: cbpts.testSSHPrivateKeyPath,
IPAddress: "192.168.1.1",
Username: "testuser",
PrivateKeyPath: cbpts.testSSHPrivateKeyPath,
}

p, err := provision.NewProvisioner(config)
Expand Down Expand Up @@ -361,9 +361,9 @@ func (cbpts *CmdBetaProvisionTestSuite) TestProvisionerLowLevelFailure() {

// Create a provisioner with test configuration
config := &provision.NodeConfig{
IPAddress: "192.168.1.100",
Username: "testuser",
PrivateKey: "/path/to/key",
IPAddress: "192.168.1.100",
Username: "testuser",
PrivateKeyPath: "/path/to/key",
}

testMachine, err := models.NewMachine(
Expand Down
24 changes: 1 addition & 23 deletions cmd/beta/provision/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ package provision

import (
"fmt"
"net"
"os"

"github.com/spf13/cobra"
)
Expand All @@ -26,33 +24,13 @@ For compute nodes, an orchestrator IP must be provided.`,
"IP address of the target node (IPv4 or IPv6 format)")
cmd.Flags().StringVar(&config.Username, "user", "",
"SSH username for authentication")
cmd.Flags().StringVar(&config.PrivateKey, "key", "",
cmd.Flags().StringVar(&config.PrivateKeyPath, "key", "",
"Path to SSH private key file (PEM format)")
cmd.Flags().StringVar(&config.OrchestratorIP, "orchestrator", "",
"IP address of the orchestrator node (required for compute nodes)")
cmd.Flags().StringVar(&config.BacalhauSettingsPath, "bacalhau-settings", "",
"Path to JSON file containing Bacalhau settings")

// Validate IP address
if net.ParseIP(config.IPAddress) == nil {
fmt.Printf("Invalid IP address: %s", config.IPAddress)
return cmd
}

// Validate orchestrator IP if provided
if config.OrchestratorIP != "" && net.ParseIP(config.OrchestratorIP) == nil {
fmt.Printf("Invalid orchestrator IP address: %s", config.OrchestratorIP)
return cmd
}

// Validate private key file path
if _, err := os.Stat(config.PrivateKey); os.IsNotExist(err) {
fmt.Printf("Private key file does not exist: %s", config.PrivateKey)
return cmd
}
cmd.Flags().BoolVar(&testMode, "test", false,
"Run in test mode (simulation only)")

// Mark required flags
err := cmd.MarkFlagRequired("ip")
if err != nil {
Expand Down
41 changes: 23 additions & 18 deletions cmd/beta/provision/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,53 +16,58 @@ const (

// NodeConfig holds the configuration for a node to be provisioned
type NodeConfig struct {
Name string `json:"name" yaml:"name"`
IPAddress string `json:"ip_address" yaml:"ip_address"`
Username string `json:"username" yaml:"username"`
PrivateKey string `json:"private_key" yaml:"private_key"`
OrchestratorIP string `json:"orchestrator_ip,omitempty" yaml:"orchestrator_ip,omitempty"`
Name string `json:"name" yaml:"name"`
IPAddress string `json:"ip_address" yaml:"ip_address"`
Username string `json:"username" yaml:"username"`
PrivateKeyPath string `json:"private_key_path" yaml:"private_key_path"`
OrchestratorIP string `json:"orchestrator_ip,omitempty" yaml:"orchestrator_ip,omitempty"`
BacalhauSettingsPath string `json:"bacalhau_settings_path,omitempty" yaml:"bacalhau_settings_path,omitempty"`
CustomScriptPath string `json:"custom_script_path,omitempty" yaml:"custom_script_path,omitempty"`
CustomScriptPath string `json:"custom_script_path,omitempty" yaml:"custom_script_path,omitempty"`
}

// Validate checks if the configuration is valid
func (c *NodeConfig) Validate() error {
func (c *NodeConfig) Validate() []error {
var errors []error

if c.IPAddress == "" {
return fmt.Errorf("IP address is required")
errors = append(errors, fmt.Errorf("IP address is required"))
}
// Validate IP address format
if net.ParseIP(c.IPAddress) == nil {
return fmt.Errorf("invalid IP address format: %s", c.IPAddress)
errors = append(errors, fmt.Errorf("invalid IP address format: %s", c.IPAddress))
}

if c.Username == "" {
return fmt.Errorf("username is required")
errors = append(errors, fmt.Errorf("username is required"))
}
if c.PrivateKey == "" {
return fmt.Errorf("private key is required")
if c.PrivateKeyPath == "" {
errors = append(errors, fmt.Errorf("private key path is required"))
}

// Verify private key exists and is readable
if _, err := os.Stat(c.PrivateKey); err != nil {
return fmt.Errorf("private key file error: %w", err)
if _, err := os.Stat(c.PrivateKeyPath); err != nil {
errors = append(errors, fmt.Errorf("private key file error: %w", err))
}

// Validate optional file paths if provided
if c.BacalhauSettingsPath != "" {
if _, err := os.Stat(c.BacalhauSettingsPath); err != nil {
return fmt.Errorf("bacalhau settings file error: %w", err)
errors = append(errors, fmt.Errorf("bacalhau settings file error: %w", err))
}
}
if c.CustomScriptPath != "" {
if _, err := os.Stat(c.CustomScriptPath); err != nil {
return fmt.Errorf("custom script file error: %w", err)
errors = append(errors, fmt.Errorf("custom script file error: %w", err))
}
}

// Validate compute node requirements
if c.OrchestratorIP != "" && net.ParseIP(c.OrchestratorIP) == nil {
return fmt.Errorf("invalid orchestrator IP address format: %s", c.OrchestratorIP)
errors = append(
errors,
fmt.Errorf("invalid orchestrator IP address format: %s", c.OrchestratorIP),
)
}

return nil
return errors
}
Loading

0 comments on commit 74ec3ed

Please sign in to comment.