Merge remote-tracking branch 'origin/main' into remove-legacy-utils

.cspell/custom-dictionary.txt

@@ -355,6 +355,7 @@ wasmlogs
 wasmmodels
 wazero
 wdbaruni's
+simonwo
 webui
 wesbos
 winderresearch
@@ -371,3 +372,63 @@ yyyymmddhhmm
 zarr
 zerolog
 zidane
+IMDC
+kvstore
+unmarshalling
+Nowf
+pkey
+machineid
+bacerror
+Nacked
+pqueue
+Routez
+Connz
+Subsz
+nuid
+Noticef
+Warnf
+Debugf
+Tracef
+sresource
+Syncer
+mathgo
+providables
+JSONV
+Idxs
+boltdblib
+hclog
+THAMTShard
+mergo
+serde
+qdisc
+puuid
+pkgs
+pscbin
+rocm
+strg
+otlploggrpc
+yacspin
+APITLSCA
+APITLSCA
+Milli
+Errf
+doesn
+cicd
+nvme
+fdisk
+mdstat
+xcom
+Fooco
+Eventuallyf
+Truef
+sekret
+Equalf
+Doesnt
+HAMT
+dagpb
+Berdly
+frrist
+swaggo
+isbadactor
+installationid
+firstbacalhauimage

.golangci.yml

@@ -54,12 +54,6 @@ linters-settings:
       - shadow
   lll:
     line-length: 140
-  misspell:
-    locale: US
-    ignore-words:
-    - favour
-    - cancelled
-    - cancelling
   nolintlint:
     allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
     allow-unused: true # report any unused nolint directives
@@ -89,7 +83,6 @@ linters:
   - govet
   - ineffassign
   - lll
-  - misspell
   - mnd
   - nakedret
   - noctx

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
   - id: detect-aws-credentials
     args: [--allow-missing-credentials]
   - id: detect-private-key
-    exclude: testdata/.*
+    exclude: 'testdata/.*|test-integration/certificates/.*'
   - id: check-yaml
   - id: check-json
 - repo: https://github.com/astral-sh/ruff-pre-commit

DESIGN.md

@@ -103,16 +103,11 @@ Ideally, we will also allow much more fine-grained control, specifying location,
   - She has a file `process.py` which includes the python code necessary to execute in a function called 'downscale()' which takes a file handle to local, processes it, and returns a bytestream.
   - She executes the following command:
 ```
-ifps job submit -f process.py -r requirements.txt -c QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
+ipfs job submit -f process.py -r requirements.txt -c QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
 ```
   - This runs the command in a local executor, first installing all the python packages necessary, and then executing them, on the subset of data available on that node.
   - Once complete, the system returns the CID of the updated dataset that she can download.
 
-- **SCENARIO 3** Want to burst to cloud but cannot move entire dataset in short time
-  - DHASH CAN YOU HELP FLESH OUT
-  - **PUSH COMPUTE INTO GENE SEQUENCER**
-  - **PIPE TO S3**
-
 ## Components to Build
 
 - Build an application that listens for jobs over NATS, receives payment somehow, runs the job in {kubernetes, docker, idk}, and returns the result to the use (ideally the 'result' is in the form of an ipfs object and we can just return the hash).
@@ -125,13 +120,3 @@ ifps job submit -f process.py -r requirements.txt -c QmbWqxBEKC3P8tqsKc98xmWNzrz
 Bacalhau means cod (the fish) in Portuguese (where several folks were brainstorming this topic).
 
 Compute-Over-Data == Cod == Bacalhau
-
-## Prior Art / Parallel Projects
-* IPFS-FAN - distributed serverless - https://research.protocol.ai/publications/ipfs-fan-a-function-addressable-computation-network/delarocha2021a.pdf
-* IPLS : A Framework for Decentralized Federated Learning- https://arxiv.org/pdf/2101.01901v1.pdf
-* Interplanetary Distributed Computing (2018) - https://github.com/yenkuanlee/IPDC
-* IPTF - IPFS + TensorFlow (2018) - https://github.com/tesserai/iptf
-* Lurk -> Run queries over Filecoin Sealed Data (no public paper yet)
-* Radix - Nomad based scheduler for  IPFS cluster (only) - high level spec doc https://docs.google.com/document/d/18hdYBmDlvusEOQ-iSNIO_IAEOvJVFL1MyAU_B8hON9Q/edit?usp=sharing
-* Bringing Arbitrary Compute to Authoritative Data https://queue.acm.org/detail.cfm?id=2645649
-* Manta: a scalable, distributed object store https://github.com/joyent/manta

Makefile

@@ -185,10 +185,10 @@ release-bacalhau-flyte: resolve-earthly
 # Target: build
 ################################################################################
 .PHONY: build
-build: resolve-earthly build-bacalhau build-plugins
+build: resolve-earthly build-bacalhau
 
 .PHONY: build-ci
-build-ci: build-bacalhau install-plugins
+build-ci: build-bacalhau
 
 .PHONY: build-dev
 build-dev: build-ci
@@ -208,7 +208,7 @@ build-webui: resolve-earthly
 ################################################################################
 # Target: build-bacalhau
 ################################################################################
-${BINARY_PATH}: build-bacalhau build-plugins
+${BINARY_PATH}: build-bacalhau
 
 .PHONY: build-bacalhau
 build-bacalhau: binary-web binary
@@ -306,7 +306,7 @@ images: docker/.pulled
 # Target: clean
 ################################################################################
 .PHONY: clean
-clean: clean-plugins
+clean:
 	${GO} clean
 	${RM} -r bin/*
 	${RM} -r webui/build/*
@@ -385,14 +385,6 @@ devstack-250:
 devstack-20:
 	go run . devstack --compute-nodes 20
 
-.PHONY: devstack-noop
-devstack-noop:
-	go run . devstack --noop
-
-.PHONY: devstack-noop-100
-devstack-noop-100:
-	go run . devstack --noop --compute-nodes 100
-
 .PHONY: devstack-race
 devstack-race:
 	go run -race . devstack
@@ -476,46 +468,6 @@ security:
 release: build-bacalhau
 	cp bin/bacalhau .
 
-ifeq ($(OS),Windows_NT)
-    detected_OS := Windows
-else
-    detected_OS := $(shell sh -c 'uname 2>/dev/null || echo Unknown')
-endif
-
-# TODO make the plugin path configurable instead of using the bacalhau config path.
-BACALHAU_CONFIG_PATH := $(shell echo $$BACALHAU_PATH)
-INSTALL_PLUGINS_DEST := $(if $(BACALHAU_CONFIG_PATH),$(BACALHAU_CONFIG_PATH)plugins/,~/.bacalhau/plugins/)
-
-EXECUTOR_PLUGINS := $(wildcard ./pkg/executor/plugins/executors/*/.)
-
-# TODO fix install on windows
-ifeq ($(detected_OS),Windows)
-    build-plugins clean-plugins install-plugins:
-	@echo "Skipping executor plugins on Windows"
-else
-    build-plugins: plugins-build
-    clean-plugins: plugins-clean
-    install-plugins: plugins-install
-
-    .PHONY: plugins-build $(EXECUTOR_PLUGINS)
-
-    plugins-build: $(EXECUTOR_PLUGINS)
-	@echo "Building executor plugins..."
-	@$(foreach plugin,$(EXECUTOR_PLUGINS),$(MAKE) --no-print-directory -C $(plugin) &&) true
-
-    .PHONY: plugins-clean $(addsuffix .clean,$(EXECUTOR_PLUGINS))
-
-    plugins-clean: $(addsuffix .clean,$(EXECUTOR_PLUGINS))
-	@echo "Cleaning executor plugins..."
-	@$(foreach plugin,$(addsuffix .clean,$(EXECUTOR_PLUGINS)),$(MAKE) --no-print-directory -C $(basename $(plugin)) clean &&) true
-
-    .PHONY: plugins-install $(addsuffix .install,$(EXECUTOR_PLUGINS))
-
-    plugins-install: plugins-build $(addsuffix .install,$(EXECUTOR_PLUGINS))
-	@echo "Installing executor plugins..."
-	@$(foreach plugin,$(addsuffix .install,$(EXECUTOR_PLUGINS)),mkdir -p $(INSTALL_PLUGINS_DEST) && cp $(basename $(plugin))/bin/* $(INSTALL_PLUGINS_DEST) &&) true
-endif
-
 .PHONY: spellcheck-code
-spellcheck-code:  ## Runs a spellchecker over all code - MVP just does one file
-	cspell -c .cspell-code.json lint ./pkg/authn/**
+spellcheck-code:
+	cspell lint  -c cspell.yaml --quiet "**/*.{go,js,ts,jsx,tsx,md,yml,yaml,json}"

clients/python/docs/OrchestratorApi.md

@@ -35,7 +35,7 @@ from pprint import pprint
 api_instance = bacalhau_apiclient.OrchestratorApi()
 id = 'id_example' # str | ID to get the job for
 include = 'include_example' # str | Takes history and executions as options. If empty will not include anything else. (optional)
-limit = 56 # int | Number of history or exeuctions to fetch. Should be used in conjugation with include (optional)
+limit = 56 # int | Number of history or executions to fetch. Should be used in conjugation with include (optional)
 
 try:
     # Returns a job.
@@ -47,11 +47,11 @@ except ApiException as e:
 
 ### Parameters
 
-Name | Type | Description  | Notes
-------------- | ------------- | ------------- | -------------
- **id** | **str**| ID to get the job for | 
- **include** | **str**| Takes history and executions as options. If empty will not include anything else. | [optional] 
- **limit** | **int**| Number of history or exeuctions to fetch. Should be used in conjugation with include | [optional] 
+Name | Type | Description                                                                          | Notes
+------------- | ------------- |--------------------------------------------------------------------------------------| -------------
+ **id** | **str**| ID to get the job for                                                                | 
+ **include** | **str**| Takes history and executions as options. If empty will not include anything else.    | [optional] 
+ **limit** | **int**| Number of history or executions to fetch. Should be used in conjugation with include | [optional] 
 
 ### Return type
 

cmd/cli/agent/alive_test.go

@@ -30,7 +30,7 @@ func (s *AliveSuite) TestAliveJSONOutput() {
 
 	aliveInfo := &apimodels.IsAliveResponse{}
 	err = marshaller.JSONUnmarshalWithMax([]byte(out), &aliveInfo)
-	s.Require().NoError(err, "Could not unmarshall the output into json - %+v", err)
+	s.Require().NoError(err, "Could not unmarshal the output into json - %+v", err)
 	s.Require().True(aliveInfo.IsReady())
 }
 
@@ -40,6 +40,6 @@ func (s *AliveSuite) TestAliveYAMLOutput() {
 
 	aliveInfo := &apimodels.IsAliveResponse{}
 	err = marshaller.YAMLUnmarshalWithMax([]byte(out), &aliveInfo)
-	s.Require().NoError(err, "Could not unmarshall the output into yaml - %+v", out)
+	s.Require().NoError(err, "Could not unmarshal the output into yaml - %+v", out)
 	s.Require().True(aliveInfo.IsReady())
 }

cmd/cli/agent/node_test.go

@@ -28,7 +28,7 @@ func (s *NodeSuite) TestNodeJSONOutput() {
 
 	nodeInfo := &models.NodeState{}
 	err = marshaller.JSONUnmarshalWithMax([]byte(out), &nodeInfo)
-	s.Require().NoError(err, "Could not unmarshall the output into json - %+v", out)
+	s.Require().NoError(err, "Could not unmarshal the output into json - %+v", out)
 	s.Require().Equal(s.Node.ID, nodeInfo.Info.ID(), "Node ID does not match in json.")
 }
 
@@ -38,6 +38,6 @@ func (s *NodeSuite) TestNodeYAMLOutput() {
 
 	nodeInfo := &models.NodeState{}
 	err = marshaller.YAMLUnmarshalWithMax([]byte(out), &nodeInfo)
-	s.Require().NoError(err, "Could not unmarshall the output into yaml - %+v", out)
+	s.Require().NoError(err, "Could not unmarshal the output into yaml - %+v", out)
 	s.Require().Equal(s.Node.ID, nodeInfo.Info.ID(), "Node ID does not match in yaml.")
 }

cmd/cli/agent/version_test.go

@@ -45,7 +45,7 @@ func (s *VersionSuite) TestVersionJSONOutput() {
 	expectedVersion := version.Get()
 	printedVersion := &models.BuildVersionInfo{}
 	err = marshaller.JSONUnmarshalWithMax([]byte(out), &printedVersion)
-	s.Require().NoError(err, "Could not unmarshall the output into json - %+v", out)
+	s.Require().NoError(err, "Could not unmarshal the output into json - %+v", out)
 	s.Require().Equal(expectedVersion, printedVersion, "Versions do not match in json.")
 }
 
@@ -58,6 +58,6 @@ func (s *VersionSuite) TestVersionYAMLOutput() {
 	expectedVersion := version.Get()
 	printedVersion := &models.BuildVersionInfo{}
 	err = marshaller.YAMLUnmarshalWithMax([]byte(out), &printedVersion)
-	s.Require().NoError(err, "Could not unmarshall the output into yaml - %+v", out)
+	s.Require().NoError(err, "Could not unmarshal the output into yaml - %+v", out)
 	s.Require().Equal(expectedVersion, printedVersion, "Versions do not match in yaml.")
 }

cmd/cli/deprecated/exec.go

@@ -0,0 +1,17 @@
+package deprecated
+
+import (
+	"github.com/spf13/cobra"
+)
+
+func NewExecCommand() *cobra.Command {
+	cancelCmd := &cobra.Command{
+		Use:        "exec",
+		Deprecated: "exec was an experimental feature and no longer supported",
+		RunE: func(cmd *cobra.Command, cmdArgs []string) error {
+			return nil
+		},
+	}
+
+	return cancelCmd
+}