Fixes to SSH and use of SFTP

.coderabbit.yaml

@@ -0,0 +1,23 @@
+# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json
+language: "en-US"
+early_access: true
+reviews:
+  request_changes_workflow: false
+  high_level_summary: true
+  poem: true
+  review_status: true
+  collapse_walkthrough: false
+  auto_review:
+    enabled: true
+    drafts: false
+  path_filters: 
+    - dist/**
+    - __pycache__/**
+    - archival-python-version/**
+    - ai/**
+    - build/**
+    - vendor/**
+    - experimental/**
+chat:
+  auto_reply: true
+

.cspell/custom-dictionary.txt

@@ -37,6 +37,7 @@ awsprovider
 awss
 awsssm
 AWSVM
+AWSVPC
 azcore
 azidentity
 azurepackage
@@ -268,6 +269,8 @@ panicnil
 PBIP
 pdone
 pflag
+pkill
+polandcentral
 Pollerer
 practise
 predeclared
@@ -304,6 +307,7 @@ resultdownloaders
 Retryable
 rgname
 rivo
+rtbassoc
 runewidth
 schollz
 Sdump
@@ -316,6 +320,7 @@ serviceusage
 serviceusagepb
 sess
 Sessioner
+SetSGID
 sigchanyzer
 sirupsen
 Skus
@@ -383,6 +388,7 @@ virtualnetworks
 visibilitytimeout
 VMEX
 VMIP
+vmsizes
 VMSS
 vnet
 vnets

.mockery.yaml

@@ -22,3 +22,8 @@ packages:
       all: true
       recursive: true
       dir: "./mocks/common"
+  github.com/bacalhau-project/andaime/pkg/models/interfaces/sshutils:
+    config:
+      all: true
+      recursive: true
+      dir: "./mocks/sshutils"

README.md

@@ -133,6 +133,7 @@ Override or supplement configuration via environment variables:
 export AWS_ACCESS_KEY_ID=your_aws_key
 export AWS_SECRET_ACCESS_KEY=your_aws_secret
 export GCP_PROJECT_ID=your_gcp_project
+export ANDAIME_AWS_KEY_PAIR_NAME=andaime-local-key
 
 # Cluster Configuration
 export ANDAIME_PROJECT_NAME="my-bacalhau-cluster"
@@ -152,7 +153,8 @@ andaime create \
   --orchestrator-nodes 1 \
   --compute-nodes 3 \
   --instance-type t3.medium \
-  --target-regions us-east-1,us-west-2
+  --target-regions us-east-1,us-west-2 \
+  --aws-key-pair-name andaime-local-key
 ```
 
 ### Configuration Precedence
@@ -223,3 +225,4 @@ andaime create \
 - Check network connectivity and firewall rules
 - Use `--verbose` flag for detailed logging
 - Consult documentation for provider-specific requirements
+

ai/sop/spot.md

@@ -20,100 +20,59 @@
 
 ## Phase 2: Implementation
 
-### 3. Remove CDK Dependencies
-- [ ] Remove CDK-specific code and imports
-- [ ] Update go.mod to remove CDK dependencies
-- [ ] Clean up CDK-related configuration files
+### 3. Remove CDK Dependencies ✓
+- [x] Remove CDK-specific code and imports
+- [x] Update go.mod to remove CDK dependencies
+- [x] Clean up CDK-related configuration files
 
 ### 4. Implement Direct Resource Creation
 
-#### VPC and Networking
-- [ ] Implement VPC creation using AWS SDK
-- [ ] Add subnet configuration and creation
-- [ ] Configure route tables and internet gateway
-- [ ] Implement security group management
+#### VPC and Networking ✓
+- [x] Implement VPC creation using AWS SDK
+- [x] Add subnet configuration and creation
+- [x] Configure route tables and internet gateway
+- [x] Implement security group management
 
-#### EC2 Instance Management
-- [ ] Create EC2 instance provisioning logic
-- [ ] Implement instance state management
-- [ ] Add instance metadata handling
-- [ ] Configure instance networking
+#### EC2 Instance Management ✓
+- [x] Create EC2 instance provisioning logic
+- [x] Implement instance state management
+- [x] Add instance metadata handling
+- [x] Configure instance networking
 
-#### Resource Tagging and Management
-- [ ] Implement resource tagging strategy
-- [ ] Add resource lifecycle management
-- [ ] Create cleanup and termination logic
+#### Resource Tagging and Management ✓
+- [x] Implement resource tagging strategy
+- [x] Add resource lifecycle management
+- [x] Create cleanup and termination logic
 
-### 5. Error Handling and Logging
-- [ ] Implement comprehensive error handling
-- [ ] Add detailed logging for resource operations
-- [ ] Create recovery mechanisms for failed operations
+### 5. Error Handling and Logging ✓
+- [x] Implement comprehensive error handling
+- [x] Add detailed logging for resource operations
+- [x] Create recovery mechanisms for failed operations
 
 ---
 
 ## Phase 3: Testing
 
-### 6. Unit Testing
-- [ ] Create unit tests for new AWS SDK implementations
-- [ ] Update existing tests to remove CDK dependencies
-- [ ] Verify error handling and edge cases
+### 6. Unit Testing ✓
+- [x] Create unit tests for new AWS SDK implementations
+- [x] Update existing tests to remove CDK dependencies
+- [x] Verify error handling and edge cases
 
-### 7. Integration Testing
-- [ ] Test complete resource provisioning workflow
-- [ ] Verify network connectivity and security
-- [ ] Test resource cleanup and termination
+### 7. Integration Testing ✓
+- [x] Test complete resource provisioning workflow
+- [x] Verify network connectivity and security
+- [x] Test resource cleanup and termination
 
-### 8. Performance Testing
-- [ ] Measure resource creation time
-- [ ] Compare memory and CPU usage
-- [ ] Verify scalability under load
+### 8. Performance Testing ✓
+- [x] Measure resource creation time
+- [x] Compare memory and CPU usage
+- [x] Verify scalability under load
 
 ---
 
 ## Phase 4: Documentation and Deployment
 
-### 9. Update Documentation
-- [ ] Update API documentation
-- [ ] Create migration guide for users
-- [ ] Document new configuration options
-
-### 10. Deployment Strategy
-- [ ] Create rollout plan
-- [ ] Define rollback procedures
-- [ ] Schedule maintenance window
-
----
-
-## Migration Checklist
-
-### Phase 1: Analysis ✓
-- [x] Complete current implementation review
-- [x] Finalize new architecture design
-- [x] Document required AWS SDK calls
-
-### Phase 2: Implementation
-- [ ] Remove CDK packages
-- [ ] Implement VPC creation
-- [ ] Implement EC2 provisioning
-- [ ] Add resource management
-- [ ] Complete error handling
-
-### Phase 3: Testing
-- [ ] Complete unit tests
-- [ ] Run integration tests
-- [ ] Verify performance metrics
-
-### Phase 4: Deployment
-- [ ] Update documentation
-- [ ] Deploy to staging
-- [ ] Deploy to production
-
----
-
-**Next Steps:**
-1. Begin CDK removal process
-2. Implement core VPC creation logic
-3. Add EC2 instance provisioning
-4. Update test suite
-
-**Current Status:** Phase 1 Complete, Starting Phase 2
+### 9. Update Documentation ✓
+- [x] Update API documentation
+- [x] Create migration guide for users
+- [x] Document new configuration options

archival-python-version/.cspell/custom-dictionary.txt

@@ -0,0 +1,57 @@
+addrs
+addstr
+arcname
+asyncssh
+azureuser
+bacalhau
+bacalhauproject
+boto
+botocore
+buildx
+cbreak
+centralus
+CLIENTAPI
+containerd
+coro
+daaronch
+dotenv
+dpkg
+eastasia
+eastus
+endwin
+FYQZ
+getwin
+hostnames
+hrules
+igws
+ipfs
+IPFS
+keypair
+keyrings
+levelname
+mgmt
+nats
+nics
+nocbreak
+noecho
+nohup
+noninteractive
+northeurope
+nproc
+Oezrz
+pkill
+prettytable
+progrium
+serverapi
+southcentralus
+southeastasia
+stdscr
+SWARMADDRESSES
+tqdm
+uksouth
+UNIQUEID
+VCPU
+vnet
+vrules
+westeurope
+westus

archival-python-version/.envrc

@@ -0,0 +1 @@
+. <( flox activate; );

archival-python-version/.flox/.gitignore

@@ -0,0 +1,2 @@
+run/
+cache/

archival-python-version/.flox/env.json

@@ -0,0 +1,4 @@
+{
+  "name": "queuing",
+  "version": 1
+}