feat(extensions): install from external registries

CHANGELOG.md

@@ -4,6 +4,80 @@
 
 ## DFX
 
+### feat: dfx extension can now be installed from external registries
+
+Modifies the `dfx extension install` command to take a flag `--registry` that allows installing extensions from external registries. Here is an example:
+
+```bash
+dfx extension install your_extension --registry=https://raw.githubusercontent.com/your-org/your-repo/master/dfx-extensions-registry.json
+```
+
+The registry file is a JSON file that contains a list of extensions. Here is an example:
+
+```json
+{
+ "compatibility": {
+ "0.15.0": {
+ "your_extension": {
+ "versions": ["0.1.0"]
+ }
+ }
+ },
+ "extensions": {
+ "your_extension": {
+ "0.1.0": {
+ "homepage": "https://github.com/your-org/your-repo",
+ "authors": "Your Org",
+ "summary": "Your extension",
+ "categories": ["development"],
+ "keywords": ["cli-helper"],
+ "description": "A longer description.",
+ "subcommands": {
+ "do_something": {
+ "about": "does something",
+ "args": {
+ "the_param": {
+ "about": "some paramater",
+ "long": "the-param"
+ }
+ }
+ }
+ },
+ "binaries": {
+ "unknown-linux-gnu-x86_64": {
+ "url": "https://raw.githubusercontent.com/your-org/your-repo/master/your_extension-v0.1.0-x86_64-unknown-linux-gnu.tar.gz",
+ "sha256": "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"
+ },
+ "apple-darwin-x86_64": {
+ "url": "https://raw.githubusercontent.com/your-org/your-repo/master/your_extension-v0.1.0-x86_64-apple-darwin.tar.gz ",
+ "sha256": "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"
+ },
+ "apple-darwin-aarch64": {
+ "url": "https://raw.githubusercontent.com/your-org/your-repo/master/your_extension-v0.1.0-aarch64-apple-darwin.tar.gz ",
+ "sha256": "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"
+ }
+ }
+ }
+ }
+ }
+}
+```
+
+The extension must be published as a tarball (`.tag.gz`), which contains a directory named `<EXTENSION_NAME>-v<EXTENSION_VERSION>-<CPU_ARCHITECTURE>-<PLATFORM>`, which contains a binary named `<EXTENSION_NAME>`, here is minimal example:
+
+```bash
+mkdir test_extension-v0.1.0-x86_64-unknown-linux-gnu
+cat > test_extension-v0.1.0-x86_64-unknown-linux-gnu/test_extension << "EOF"
+#!/usr/bin/env bash
+
+echo Hello, World!
+EOF
+tar -czf test_extension-v0.1.0-x86_64-unknown-linux-gnu.tar.gz test_extension-v0.1.0-x86_64-unknown-linux-gnu
+# you'll also need to generate a sha256 hash of the tarball and put it into the registry file
+shasum -a 256 test_extension-v0.1.0-x86_64-unknown-linux-gnu.tar.gz 
+```
+
+
 ### feat!: Removed dfx nns and dfx sns commands
 
 Both have now been turned into the dfx extensions. In order to obtain them, please run `dfx extension install nns` and `dfx extension install sns` respectively. After the installation, you can use them as you did before: `dfx nns ...`, and `dfx sns ...`.

e2e/tests-dfx/extension.bash

@@ -209,3 +209,80 @@ EOF
  assert_command dfx extension run test_extension abc --the-another-param 464646 --the-param 123 456 789
  assert_eq "abc --the-another-param 464646 --the-param 123 456 789 --dfx-cache-path $CACHE_DIR"
 }
+
+@test "install extension from external registry" {
+ # strip semver suffix
+ DFX_VERSION=$(dfx --version | sed 's/-.*//' | cut -d ' ' -f 2)
+ # find unoocupied port
+ port=49152
+ while lsof -i :$port > /dev/null 2>&1
+ do
+ port=$((port+1))
+ done
+
+ mkdir test_extension-v0.1.0-x86_64-unknown-linux-gnu
+ cat > test_extension-v0.1.0-x86_64-unknown-linux-gnu/test_extension << "EOF"
+#!/usr/bin/env bash
+
+echo $@
+EOF
+ cp -r test_extension-v0.1.0-x86_64-unknown-linux-gnu test_extension-v0.1.0-aarch64-apple-darwin
+ cp -r test_extension-v0.1.0-x86_64-unknown-linux-gnu test_extension-v0.1.0-x86_64-apple-darwin
+ tar -czf test_extension-v0.1.0-x86_64-unknown-linux-gnu.tar.gz test_extension-v0.1.0-x86_64-unknown-linux-gnu
+ tar -czf test_extension-v0.1.0-aarch64-apple-darwin.tar.gz test_extension-v0.1.0-aarch64-apple-darwin
+ tar -czf test_extension-v0.1.0-x86_64-apple-darwin.tar.gz test_extension-v0.1.0-x86_64-apple-darwin
+ cat > registry.json <<EOF
+{
+ "compatibility": {
+ "$DFX_VERSION": {
+ "test_extension": {
+ "versions": ["0.1.0"]
+ }
+ }
+ },
+ "extensions": {
+ "test_extension": {
+ "0.1.0": {
+ "homepage": "https://github.com/dfinity/sdk",
+ "authors": "DFINITY",
+ "summary": "A foo extension",
+ "categories": ["development"],
+ "keywords": ["development"],
+ "description": "A longer description.",
+ "subcommands": {},
+ "binaries": {
+ "unknown-linux-gnu-x86_64": {
+ "url": "http://localhost:$port/test_extension-v0.1.0-x86_64-unknown-linux-gnu.tar.gz",
+ "sha256": "$(shasum -a 256 test_extension-v0.1.0-x86_64-unknown-linux-gnu.tar.gz | cut -d ' ' -f 1)"
+ },
+ "apple-darwin-x86_64": {
+ "url": "http://localhost:$port/test_extension-v0.1.0-x86_64-apple-darwin.tar.gz ",
+ "sha256": "$(shasum -a 256 test_extension-v0.1.0-x86_64-apple-darwin.tar.gz | cut -d ' ' -f 1)"
+ },
+ "apple-darwin-aarch64": {
+ "url": "http://localhost:$port/test_extension-v0.1.0-aarch64-apple-darwin.tar.gz ",
+ "sha256": "$(shasum -a 256 test_extension-v0.1.0-aarch64-apple-darwin.tar.gz | cut -d ' ' -f 1)"
+ }
+ }
+ }
+ }
+ }
+}
+EOF
+
+ python3 -m http.server "$port" &
+ pid=$!
+
+ # wait until the server is up
+ while ! echo exit | nc localhost "$port"; do sleep 1; done
+ echo "Server is up"
+ echo yes | (
+ assert_command dfx extension install test_extension --registry "http://localhost:$port/registry.json"
+ )
+ kill $pid
+
+ assert_command dfx extension list
+ assert_match "test_extension"
+ assert_command dfx test_extension
+ assert_eq "--dfx-cache-path $(dfx cache show)"
+}

src/dfx-core/Cargo.toml

@@ -33,12 +33,13 @@ sec1 = { workspace = true, features = ["std"] }
 semver = { workspace = true, features = ["serde"] }
 serde.workspace = true
 serde_json.workspace = true
+sha2.workspace = true
 slog = { workspace = true, features = ["max_level_trace"] }
 tar.workspace = true
 tempfile.workspace = true
 thiserror.workspace = true
 tiny-bip39 = "1.0.0"
-url.workspace = true
+url = { workspace = true, features = [ "serde" ] }
 
 [dev-dependencies]
 proptest = "1.0"

src/dfx-core/src/error/extension.rs

@@ -4,6 +4,9 @@ use thiserror::Error;
 
 #[derive(Error, Debug)]
 pub enum ExtensionError {
+ #[error(transparent)]
+ Io(#[from] crate::error::unified_io::UnifiedIoError),
+
  // errors related to extension directory management
  #[error("Cannot find cache directory: '{0}'")]
  FindCacheDirectoryFailed(crate::error::cache::CacheError),
@@ -54,12 +57,28 @@ pub enum ExtensionError {
  #[error("Cannot create temporary directory at '{0}': {1}")]
  CreateTemporaryDirectoryFailed(std::path::PathBuf, std::io::Error),
 
- #[error(transparent)]
- Io(#[from] crate::error::fs::FsError),
+ #[error("Failed to download extension, because the checksum of the downloaded archive (sha256:{0}) (downloaded from: '{1}') doesn't match the one provided by the manifest (sha256:{2})")]
+ ChecksumMismatch(String, String, String),
 
  #[error("Platform '{0}' is not supported.")]
  PlatformNotSupported(String),
 
+ // errors related to installing extensions from 3rd party registries
+ #[error("Extension manifest URL '{0}' is not valid: {1}")]
+ InvalidExternalManifestUrl(String, url::ParseError),
+
+ #[error("Entry 'binaries' not found in extension manifest entry for extension '{0}' version '{1}'. Please contact the extension author.")]
+ BinaryEntryNotFoundInExtensionManifest(String, semver::Version),
+
+ #[error("Entry 'extensions.{0}' not found in extension manifest entry.")]
+ ExtensionNameNotFoundInManifest(String),
+
+ #[error("Extension '{0}' was found in the manifest in both 'comppatibility' and 'extensions' entries, however, the latest compatible version of the extension ({1}) could not be found in 'extensions' entry. Please contact the extension author.")]
+ MalformedManifestExtensionVersionNotFound(String, semver::Version),
+
+ #[error("Cannot save extension manifest: {0}")]
+ SaveExtensionManifestFailed(crate::error::structured_file::StructuredFileError),
+
  // errors related to uninstalling extensions
  #[error("Cannot uninstall extension: {0}")]
  InsufficientPermissionsToDeleteExtensionDirectory(crate::error::fs::FsError),