pack: skip unavailable configs from etcd and tcs

@TarantoolBot document
Title: pack: skip configs from etcd and tcs

This patch fixes the `tt pack` error if `etcd` or `tcs` are unavailable
during package creation.

Part of #1038

Author: patapenka-alexey

CHANGELOG.md

@@ -42,6 +42,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 - `tt coredump inspect`: fails for tarantool-ee coredump archive if the source
   directory is missing.
+- `tt pack`: fails if `etcd` or `tcs` are present in the configuration
+  and not available.
 
 ## [2.6.0] - 2024-11-29
 

cli/pack/opts.go

@@ -49,7 +49,7 @@ func initAppsInfo(cliOpts *config.CliOpts, cmdCtx *cmdcontext.CmdCtx, packCtx *P
 	}
 	packCtx.AppList = appList
 	packCtx.AppsInfo, err = running.CollectInstancesForApps(packCtx.AppList, cliOpts,
-		cmdCtx.Cli.ConfigDir, cmdCtx.Integrity, true)
+		cmdCtx.Cli.ConfigDir, cmdCtx.Integrity, false)
 	if err != nil {
 		return fmt.Errorf("failed to collect applications info: %s", err)
 	}

cli/running/running.go

@@ -348,7 +348,7 @@ func loadInstanceConfig(configPath, instName string,
 
 // collectInstancesFromAppDir collects instances information from application directory.
 func collectInstancesFromAppDir(appDir string, selectedInstName string,
-	integrityCtx integrity.IntegrityCtx, instancesScriptsRequired bool) (
+	integrityCtx integrity.IntegrityCtx, configRequired bool) (
 	[]InstanceCtx,
 	error,
 ) {
@@ -376,7 +376,7 @@ func collectInstancesFromAppDir(appDir string, selectedInstName string,
 					InstName:       filepath.Base(appDir),
 					AppDir:         appDir,
 					SingleApp:      true}}, nil
-			} else if instancesScriptsRequired {
+			} else if configRequired {
 				return nil, fmt.Errorf("require files are missing in application directory %q: "+
 					"there must be instances config or the default instance script (%q)",
 					appDir, "init.lua")
@@ -410,15 +410,15 @@ func collectInstancesFromAppDir(appDir string, selectedInstName string,
 		log.Debugf("Instance %q", instance.InstName)
 
 		if instance.Configuration, err = loadInstanceConfig(instance.ClusterConfigPath,
-			instance.InstName, integrityCtx); err != nil {
+			instance.InstName, integrityCtx); err != nil && configRequired {
 			return instances, fmt.Errorf("error loading instance %q configuration from "+
 				"config %q: %w", instance.InstName, instance.ClusterConfigPath, err)
 		}
 
 		instance.SingleApp = false
 		if instance.InstanceScript, err = findInstanceScriptInAppDir(appDir, instance.InstName,
 			appDirFiles.clusterCfgPath, appDirFiles.defaultLuaPath); err != nil &&
-			instancesScriptsRequired {
+			configRequired {
 			return instances, fmt.Errorf("cannot find instance script for %q in config %q: %w ",
 				instance.InstName, appDirFiles.clusterCfgPath, err)
 		}
@@ -434,7 +434,7 @@ func collectInstancesFromAppDir(appDir string, selectedInstName string,
 
 // CollectInstances searches all instances available in application.
 func CollectInstances(appName string, applicationsDir string,
-	integrityCtx integrity.IntegrityCtx, instancesScriptsRequired bool) ([]InstanceCtx, error) {
+	integrityCtx integrity.IntegrityCtx, configRequired bool) ([]InstanceCtx, error) {
 	// The user can select a specific instance from the application.
 	// Example: `tt status application:server`.
 	selectedInstName := ""
@@ -463,7 +463,7 @@ func CollectInstances(appName string, applicationsDir string,
 	}
 
 	return collectInstancesFromAppDir(appDir, selectedInstName, integrityCtx,
-		instancesScriptsRequired)
+		configRequired)
 }
 
 // cleanup removes runtime artifacts.
@@ -625,7 +625,7 @@ func GetClusterConfigPath(cliOpts *config.CliOpts,
 
 // CollectInstancesForApps collects instances information per application.
 func CollectInstancesForApps(appList []string, cliOpts *config.CliOpts,
-	ttConfigDir string, integrityCtx integrity.IntegrityCtx, instancesScriptsRequired bool) (
+	ttConfigDir string, integrityCtx integrity.IntegrityCtx, configRequired bool) (
 	map[string][]InstanceCtx, error) {
 	instEnabledPath := cliOpts.Env.InstancesEnabled
 	if cliOpts.Env.InstancesEnabled == "." {
@@ -635,7 +635,7 @@ func CollectInstancesForApps(appList []string, cliOpts *config.CliOpts,
 	for _, appName := range appList {
 		appName = strings.TrimSuffix(appName, ".lua")
 		collectedInstances, err := CollectInstances(appName, instEnabledPath, integrityCtx,
-			instancesScriptsRequired)
+			configRequired)
 		if err != nil {
 			return apps, fmt.Errorf("can't collect instance information for %s: %w",
 				appName, err)
@@ -676,7 +676,7 @@ func createInstanceDataDirectories(instance InstanceCtx) error {
 
 // FillCtx fills the RunningCtx context.
 func FillCtx(cliOpts *config.CliOpts, cmdCtx *cmdcontext.CmdCtx,
-	runningCtx *RunningCtx, args []string, instancesScriptsRequired bool) error {
+	runningCtx *RunningCtx, args []string, configRequired bool) error {
 	var err error
 
 	if len(args) > 1 && cmdCtx.CommandName != "run" && cmdCtx.CommandName != "connect" &&
@@ -703,7 +703,7 @@ func FillCtx(cliOpts *config.CliOpts, cmdCtx *cmdcontext.CmdCtx,
 	}
 
 	instances, err := CollectInstancesForApps(appList, cliOpts,
-		cmdCtx.Cli.ConfigDir, cmdCtx.Integrity, instancesScriptsRequired)
+		cmdCtx.Cli.ConfigDir, cmdCtx.Integrity, configRequired)
 	if err != nil {
 		return err
 	}

test/integration/pack/test_bundles/vshard_app/instances.enabled/test_app

@@ -0,0 +1 @@
+../test_app

test/integration/pack/test_bundles/vshard_app/test_app/config.yaml

@@ -0,0 +1,75 @@
+credentials:
+  users:
+    client:
+      password: 'secret'
+      roles: [super]
+    replicator:
+      password: 'secret'
+      roles: [replication]
+    storage:
+      password: 'secret'
+      roles: [sharding]
+
+iproto:
+  advertise:
+    peer:
+      login: replicator
+    sharding:
+      login: storage
+
+sharding:
+  bucket_count: 3000
+
+groups:
+  storages:
+    app:
+      module: storage
+    sharding:
+      roles: [storage]
+    replication:
+      failover: manual
+    replicasets: 
+      storage-001:
+        leader: storage-001-a
+        instances: 
+          storage-001-a:
+            iproto:
+              listen:
+                - uri: localhost:3301
+          storage-001-b:
+            iproto:
+              listen:
+                - uri: localhost:3302
+      storage-002:
+        leader: storage-002-a
+        instances: 
+          storage-002-a:
+            iproto:
+              listen:
+                - uri: localhost:3303
+          storage-002-b:
+            iproto:
+              listen:
+                - uri: localhost:3304
+  routers:
+    app:
+      module: router
+    sharding:
+      roles: [router]
+    replicasets: 
+      router-001:
+        instances: 
+          router-001-a:
+            iproto:
+              listen:
+                - uri: localhost:3305
+config:
+  etcd:
+    endpoints:
+    - http://localhost:2379
+    prefix: /test_app
+    username: client
+    password: secret
+    http:
+      request:
+        timeout: 3

test/integration/pack/test_bundles/vshard_app/test_app/instances.yaml

@@ -0,0 +1,11 @@
+---
+storage-001-a:
+
+storage-001-b:
+
+storage-002-a:
+
+storage-002-b:
+
+router-001-a:
+

test/integration/pack/test_bundles/vshard_app/test_app/router.lua

@@ -0,0 +1,34 @@
+local vshard = require('vshard')
+local log = require('log')
+
+-- Bootstrap the vshard router.
+while true do
+    local ok, err = vshard.router.bootstrap({
+        if_not_bootstrapped = true,
+    })
+    if ok then
+        break
+    end
+    log.info(('Router bootstrap error: %s'):format(err))
+end
+
+-- Put data into the cluster.
+function put(id, name, age)
+    local bucket_id = vshard.router.bucket_id_mpcrc32({id})
+    vshard.router.callrw(bucket_id, 'put', {id, bucket_id, name, age})
+end
+
+-- Get data from the cluster.
+function get(id)
+    local bucket_id = vshard.router.bucket_id_mpcrc32({id})
+    return vshard.router.callro(bucket_id, 'get', {id})
+end
+
+-- Put sample data.
+function put_sample_data()
+    put(1, 'Elizabeth', 12)
+    put(2, 'Mary', 46)
+    put(3, 'David', 33)
+    put(4, 'William', 81)
+    put(5, 'Jack', 35)
+end

test/integration/pack/test_bundles/vshard_app/test_app/storage.lua

@@ -0,0 +1,56 @@
+local vshard = require('vshard')
+
+-- Create 'customers' space.
+box.once('customers', function()
+    box.schema.create_space('customers', {
+        format = {{
+            name = 'id',
+            type = 'unsigned'
+        }, {
+            name = 'bucket_id',
+            type = 'unsigned'
+        }, {
+            name = 'name',
+            type = 'string'
+        }, {
+            name = 'age',
+            type = 'number'
+        }}
+    })
+    box.space.customers:create_index('primary_index', {
+        parts = {{
+            field = 1,
+            type = 'unsigned'
+        }}
+    })
+    box.space.customers:create_index('bucket_id', {
+        parts = {{
+            field = 2,
+            type = 'unsigned'
+        }},
+        unique = false
+    })
+    box.space.customers:create_index('age', {
+        parts = {{
+            field = 4,
+            type = 'number'
+        }},
+        unique = false
+    })
+end)
+
+-- Put data to the 'customers' space.
+-- Function should be called by the router.
+function put(id, bucket_id, name, age)
+    box.space.customers:insert({id, bucket_id, name, age})
+end
+
+-- Get data from the 'customers' space.
+-- Function should be called by the router.
+function get(id)
+    local tuple = box.space.customers:get(id)
+    if tuple == nil then
+        return nil
+    end
+    return {tuple.id, tuple.name, tuple.age}
+end

test/integration/pack/test_bundles/vshard_app/test_app/test_app-scm-1.rockspec

@@ -0,0 +1,11 @@
+package = 'test_app'
+version = 'scm-1'
+source  = {
+    url = '/dev/null',
+}
+dependencies = {
+    'vshard == 0.1.25'
+}
+build = {
+    type = 'none';
+}

test/integration/pack/test_bundles/vshard_app/tt.yaml

@@ -0,0 +1,21 @@
+env:
+  bin_dir: bin
+  inc_dir: include
+  instances_enabled: instances.enabled
+  restart_on_failure: false
+  tarantoolctl_layout: false
+modules:
+  directory: modules
+app:
+  run_dir: var/run
+  log_dir: var/log
+  wal_dir: var/lib
+  memtx_dir: var/lib
+  vinyl_dir: var/lib
+ee:
+  credential_path: ""
+templates:
+- path: templates
+repo:
+  rocks: ""
+  distfiles: distfiles

test/integration/pack/test_pack.py

@@ -55,6 +55,12 @@ def assert_single_app_env(config):
     assert config["env"]["bin_dir"] == "bin"
 
 
+def assert_vshard_app_env(config):
+    assert config["env"]["instances_enabled"] == "instances.enabled"
+    assert config["env"]["bin_dir"] == "bin"
+    assert config["env"]["inc_dir"] == "include"
+
+
 def assert_artifacts_env(config):
     assert config["app"]["wal_dir"] == "var/lib"
     assert config["app"]["vinyl_dir"] == "var/lib"
@@ -520,6 +526,36 @@ def prepare_tgz_test_cases(tt_cmd) -> list:
             ],
             "check_env": ["myapp", assert_single_app_env, assert_artifacts_env]
         },
+        {
+            "name": "Vshard app packing",
+            "bundle_src": "vshard_app",
+            "cmd": tt_cmd,
+            "pack_type": "tgz",
+            "args": ["--name", "test_app"],
+            "res_file": "test_app-0.1.0.0." + get_arch() + ".tar.gz",
+            "check_exist": [
+                os.path.join("bin", "tarantool"),
+                os.path.join("bin", "tt"),
+                os.path.join("instances.enabled", "test_app"),
+                "tt.yaml",
+                "test_app/config.yaml",
+                "test_app/instances.yaml",
+                "test_app/router.lua",
+                "test_app/storage.lua",
+            ],
+            "check_not_exist": [
+                "test_app/test_app-scm-1.rockspec",
+                "test_app/tt.yaml",
+                "include",
+                "modules",
+                "templates",
+                os.path.join("test_app", "include"),
+                os.path.join("test_app", "modules"),
+                os.path.join("test_app", "templates"),
+                os.path.join("test_app", "distfiles"),
+            ],
+            "check_env": [".", assert_vshard_app_env, assert_artifacts_env]
+        },
     ]