Speed up integration tests (kopia#653)

* testing: don't use expensive scrypt-65536-8-1 in integration tests

* testing: use platform-specific encryption and hashing for arm and arm64 to speed up tests

* testing: manually manage log directory to be able to analyze integration test failures

* testing: snapshot_gc_test was too quick

* Makefile: renamed target building integration test binary

Author: jkowalski

Makefile

@@ -193,15 +193,15 @@ test: $(gotestsum)
 vtest: $(gotestsum)
 	$(GO_TEST) -count=1 -short -v -timeout $(UNIT_TESTS_TIMEOUT) ./...
 
-dist-binary:
+build-integration-test-binary:
 	go build -o $(KOPIA_INTEGRATION_EXE) -tags testing github.com/kopia/kopia
 
 integration-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE)
-integration-tests: dist-binary $(gotestsum)
+integration-tests: build-integration-test-binary $(gotestsum)
 	 $(GO_TEST) $(TEST_FLAGS) -count=1 -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/end_to_end_test
 
 endurance-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE)
-endurance-tests: dist-binary $(gotestsum)
+endurance-tests: build-integration-test-binary $(gotestsum)
 	 $(GO_TEST) $(TEST_FLAGS) -count=1 -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/endurance_test
 
 robustness-tool-tests: $(gotestsum)
@@ -279,7 +279,7 @@ endif
 
 ifneq ($(TRAVIS_TAG),)
 
-travis-create-long-term-repository: dist-binary travis-install-cloud-sdk
+travis-create-long-term-repository: build-integration-test-binary travis-install-cloud-sdk
 	echo Creating long-term repository $(TRAVIS_TAG)...
 	KOPIA_EXE=$(KOPIA_INTEGRATION_EXE) ./tests/compat_test/gen-compat-repo.sh
 

repo/crypto_key_derivation nontest.go

@@ -0,0 +1,23 @@
+// +build !testing
+
+package repo
+
+import (
+	"github.com/pkg/errors"
+	"golang.org/x/crypto/scrypt"
+)
+
+// defaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
+const defaultKeyDerivationAlgorithm = "scrypt-65536-8-1"
+
+func (f *formatBlob) deriveMasterKeyFromPassword(password string) ([]byte, error) {
+	const masterKeySize = 32
+
+	switch f.KeyDerivationAlgorithm {
+	case "scrypt-65536-8-1":
+		return scrypt.Key([]byte(password), f.UniqueID, 65536, 8, 1, masterKeySize)
+
+	default:
+		return nil, errors.Errorf("unsupported key algorithm: %v", f.KeyDerivationAlgorithm)
+	}
+}

repo/crypto_key_derivation.go

@@ -4,26 +4,9 @@ import (
 	"crypto/sha256"
 	"io"
 
-	"github.com/pkg/errors"
 	"golang.org/x/crypto/hkdf"
-	"golang.org/x/crypto/scrypt"
 )
 
-// defaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
-const defaultKeyDerivationAlgorithm = "scrypt-65536-8-1"
-
-func (f *formatBlob) deriveMasterKeyFromPassword(password string) ([]byte, error) {
-	const masterKeySize = 32
-
-	switch f.KeyDerivationAlgorithm {
-	case "scrypt-65536-8-1":
-		return scrypt.Key([]byte(password), f.UniqueID, 65536, 8, 1, masterKeySize)
-
-	default:
-		return nil, errors.Errorf("unsupported key algorithm: %v", f.KeyDerivationAlgorithm)
-	}
-}
-
 // deriveKeyFromMasterKey computes a key for a specific purpose and length using HKDF based on the master key.
 func deriveKeyFromMasterKey(masterKey, uniqueID, purpose []byte, length int) []byte {
 	key := make([]byte, length)

repo/crypto_key_derivation_testing.go

@@ -0,0 +1,29 @@
+// +build testing
+
+package repo
+
+import (
+	"crypto/sha256"
+
+	"github.com/pkg/errors"
+)
+
+// defaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
+const defaultKeyDerivationAlgorithm = "testing-only-insecure"
+
+func (f *formatBlob) deriveMasterKeyFromPassword(password string) ([]byte, error) {
+	const masterKeySize = 32
+
+	switch f.KeyDerivationAlgorithm {
+	case defaultKeyDerivationAlgorithm:
+		h := sha256.New()
+		if _, err := h.Write([]byte(password)); err != nil {
+			return nil, err
+		}
+
+		return h.Sum(nil), nil
+
+	default:
+		return nil, errors.Errorf("unsupported key algorithm: %v", f.KeyDerivationAlgorithm)
+	}
+}

tests/end_to_end_test/all_formats_test.go

@@ -9,6 +9,16 @@ import (
 )
 
 func TestAllFormatsSmokeTest(t *testing.T) {
+	srcDir := t.TempDir()
+
+	// 3-level directory with <=10 files and <=10 subdirectories at each level
+	testenv.CreateDirectoryTree(srcDir, testenv.DirectoryTreeOptions{
+		Depth:                  2,
+		MaxSubdirsPerDirectory: 5,
+		MaxFilesPerDirectory:   5,
+		MaxFileSize:            100,
+	}, nil)
+
 	for _, encryptionAlgo := range encryption.SupportedAlgorithms(false) {
 		encryptionAlgo := encryptionAlgo
 
@@ -22,8 +32,10 @@ func TestAllFormatsSmokeTest(t *testing.T) {
 					e := testenv.NewCLITest(t)
 					defer e.RunAndExpectSuccess(t, "repo", "disconnect")
 
+					e.DefaultRepositoryCreateFlags = nil
+
 					e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--block-hash", hashAlgo, "--encryption", encryptionAlgo)
-					e.RunAndExpectSuccess(t, "snap", "create", sharedTestDataDir1)
+					e.RunAndExpectSuccess(t, "snap", "create", srcDir)
 
 					sources := e.ListSnapshotsAndExpectSuccess(t)
 					if got, want := len(sources), 1; got != want {

tests/end_to_end_test/snapshot_gc_test.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/kopia/kopia/tests/testenv"
 )
@@ -61,6 +62,9 @@ how are you
 	// data block + directory block + manifest block + manifest block from manifest deletion
 	e.RunAndVerifyOutputLineCount(t, expectedContentCount, "content", "list")
 
+	// make sure we are not too quick
+	time.Sleep(2 * time.Second)
+
 	// garbage-collect for real, this time without age limit
 	e.RunAndExpectSuccess(t, "snapshot", "gc", "--delete", "--min-age", "0s")
 

tests/testenv/cli_test_env.go

@@ -41,6 +41,8 @@ type CLITest struct {
 	fixedArgs   []string
 	Environment []string
 
+	DefaultRepositoryCreateFlags []string
+
 	PassthroughStderr bool
 
 	LogsDir string
@@ -70,7 +72,22 @@ func NewCLITest(t *testing.T) *CLITest {
 	}
 
 	configDir := t.TempDir()
-	logsDir := t.TempDir()
+
+	cleanName := strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(
+		t.Name(),
+		"/", "_"), "\\", "_"), ":", "_")
+
+	logsDir := filepath.Join(os.TempDir(), "kopia-logs", cleanName+"."+clock.Now().Local().Format("20060102150405"))
+
+	t.Cleanup(func() {
+		if t.Failed() {
+			t.Logf("logs are available in %v", logsDir)
+			return
+		}
+
+		os.RemoveAll(logsDir)
+	})
+
 	fixedArgs := []string{
 		// use per-test config file, to avoid clobbering current user's setup.
 		"--config-file", filepath.Join(configDir, ".kopia.config"),
@@ -87,13 +104,24 @@ func NewCLITest(t *testing.T) *CLITest {
 		fixedArgs = append(fixedArgs, "--no-use-keyring")
 	}
 
+	var formatFlags []string
+
+	switch runtime.GOARCH {
+	case "arm64", "arm":
+		formatFlags = []string{
+			"--encryption", "CHACHA20-POLY1305-HMAC-SHA256",
+			"--block-hash", "BLAKE2S-256",
+		}
+	}
+
 	return &CLITest{
-		startTime: clock.Now(),
-		RepoDir:   t.TempDir(),
-		ConfigDir: configDir,
-		Exe:       filepath.FromSlash(exe),
-		fixedArgs: fixedArgs,
-		LogsDir:   logsDir,
+		startTime:                    clock.Now(),
+		RepoDir:                      t.TempDir(),
+		ConfigDir:                    configDir,
+		Exe:                          filepath.FromSlash(exe),
+		fixedArgs:                    fixedArgs,
+		DefaultRepositoryCreateFlags: formatFlags,
+		LogsDir:                      logsDir,
 		Environment: []string{
 			"KOPIA_PASSWORD=" + TestRepoPassword,
 			"KOPIA_ADVANCED_COMMANDS=enabled",
@@ -117,11 +145,9 @@ func (e *CLITest) RunAndExpectSuccess(t *testing.T, args ...string) []string {
 func (e *CLITest) RunAndProcessStderr(t *testing.T, callback func(line string) bool, args ...string) *exec.Cmd {
 	t.Helper()
 
-	t.Logf("running 'kopia %v'", strings.Join(args, " "))
-	cmdArgs := append(append([]string(nil), e.fixedArgs...), args...)
-
-	c := exec.Command(e.Exe, cmdArgs...)
+	c := exec.Command(e.Exe, e.cmdArgs(args)...)
 	c.Env = append(os.Environ(), e.Environment...)
+	t.Logf("running '%v %v'", c.Path, c.Args)
 
 	stderrPipe, err := c.StderrPipe()
 	if err != nil {
@@ -184,15 +210,27 @@ func (e *CLITest) RunAndVerifyOutputLineCount(t *testing.T, wantLines int, args
 	return lines
 }
 
+func (e *CLITest) cmdArgs(args []string) []string {
+	var suffix []string
+
+	// detect repository creation and override DefaultRepositoryCreateFlags for best
+	// performance on the current platform.
+	if len(args) >= 2 && (args[0] == "repo" && args[1] == "create") {
+		suffix = e.DefaultRepositoryCreateFlags
+	}
+
+	return append(append(append([]string(nil), e.fixedArgs...), args...), suffix...)
+}
+
 // Run executes kopia with given arguments and returns the output lines.
 func (e *CLITest) Run(t *testing.T, expectedError bool, args ...string) (stdout, stderr []string, err error) {
 	t.Helper()
-	t.Logf("running '%v %v'", e.Exe, strings.Join(args, " "))
-	cmdArgs := append(append([]string(nil), e.fixedArgs...), args...)
 
-	c := exec.Command(e.Exe, cmdArgs...)
+	c := exec.Command(e.Exe, e.cmdArgs(args)...)
 	c.Env = append(os.Environ(), e.Environment...)
 
+	t.Logf("running '%v %v'", c.Path, c.Args)
+
 	errOut := &bytes.Buffer{}
 	c.Stderr = errOut