Merge pull request #448 from mozilla/dev/fix-aws-kms-enc-ctx

KMS Encryption Context bug fix, autofix, and additional testing

.travis.yml

@@ -15,7 +15,8 @@ before_install:
   - source ~/.cargo/env
 
 script:
-  - make
+  - 'if [ "$TRAVIS_REPO_SLUG" != "mozilla/sops" ]; then make; fi'
+  - 'if [ "$TRAVIS_REPO_SLUG" = "mozilla/sops" ]; then make origin-build; fi'
   - bash <(curl -s https://codecov.io/bash)
 
 before_deploy:
@@ -27,7 +28,7 @@ before_deploy:
   - GOOS=linux CGO_ENABLED=0 go build -o dist/sops-${TRAVIS_TAG}.linux go.mozilla.org/sops/cmd/sops
   - |
       if [ ! -z "$TRAVIS_TAG" ]; then
-        version="$(grep '^const version' cmd/sops/version.go |cut -d '"' -f 2)"
+        version="$(grep '^const Version' version/version.go |cut -d '"' -f 2)"
         if [ "$version" != "$TRAVIS_TAG" ]; then
             echo "Git tag $TRAVIS_TAG does not match version $version, update the source!"
             exit 1

Makefile

@@ -7,6 +7,7 @@ GO 		:= GO15VENDOREXPERIMENT=1 go
 GOLINT 		:= golint
 
 all: test vet generate install functional-tests
+origin-build: test vet generate install functional-tests-all
 
 install:
 	$(GO) install go.mozilla.org/sops/cmd/sops
@@ -40,6 +41,12 @@ functional-tests:
 	$(GO) build -o functional-tests/sops go.mozilla.org/sops/cmd/sops
 	cd functional-tests && cargo test
 
+# Ignored tests are ones that require external services (e.g. AWS KMS)
+# 	TODO: Once `--include-ignored` lands in rust stable, switch to that.
+functional-tests-all:
+	$(GO) build -o functional-tests/sops go.mozilla.org/sops/cmd/sops
+	cd functional-tests && cargo test && cargo test -- --ignored
+
 deb-pkg: install
 	rm -rf tmppkg
 	mkdir -p tmppkg/usr/local/bin

cmd/sops/common/common.go

@@ -3,17 +3,22 @@ package common
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
+	wordwrap "github.com/mitchellh/go-wordwrap"
 	"go.mozilla.org/sops"
 	"go.mozilla.org/sops/cmd/sops/codes"
 	"go.mozilla.org/sops/keyservice"
-	"go.mozilla.org/sops/stores/json"
-	"go.mozilla.org/sops/stores/yaml"
+	"go.mozilla.org/sops/kms"
 	"go.mozilla.org/sops/stores/dotenv"
 	"go.mozilla.org/sops/stores/ini"
+	"go.mozilla.org/sops/stores/json"
+	"go.mozilla.org/sops/stores/yaml"
+	"go.mozilla.org/sops/version"
+	"golang.org/x/crypto/ssh/terminal"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -26,7 +31,6 @@ type Store interface {
 	ExampleFileEmitter
 }
 
-
 // DecryptTreeOpts are the options needed to decrypt a tree
 type DecryptTreeOpts struct {
 	// Tree is the tree to be decrypted
@@ -136,3 +140,202 @@ func DefaultStoreForPath(path string) Store {
 	}
 	return &json.BinaryStore{}
 }
+
+const KMS_ENC_CTX_BUG_FIXED_VERSION = "3.3.0"
+
+func DetectKMSEncryptionContextBug(tree *sops.Tree) (bool, error) {
+	versionCheck, err := version.AIsNewerThanB(KMS_ENC_CTX_BUG_FIXED_VERSION, tree.Metadata.Version)
+	if err != nil {
+		return false, err
+	}
+
+	if versionCheck {
+		_, _, key := GetKMSKeyWithEncryptionCtx(tree)
+		if key != nil {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func GetKMSKeyWithEncryptionCtx(tree *sops.Tree) (keyGroupIndex int, keyIndex int, key *kms.MasterKey) {
+	for i, kg := range tree.Metadata.KeyGroups {
+		for n, k := range kg {
+			kmsKey, ok := k.(*kms.MasterKey)
+			if ok {
+				if kmsKey.EncryptionContext != nil && len(kmsKey.EncryptionContext) >= 2 {
+					duplicateValues := map[string]int{}
+					for _, v := range kmsKey.EncryptionContext {
+						duplicateValues[*v] = duplicateValues[*v] + 1
+					}
+					if len(duplicateValues) > 1 {
+						return i, n, kmsKey
+					}
+				}
+			}
+		}
+	}
+	return 0, 0, nil
+}
+
+type GenericDecryptOpts struct {
+	Cipher      sops.Cipher
+	InputStore  sops.Store
+	InputPath   string
+	IgnoreMAC   bool
+	KeyServices []keyservice.KeyServiceClient
+}
+
+// LoadEncryptedFileWithBugFixes is a wrapper around LoadEncryptedFile which includes
+// check for the issue described in https://github.com/mozilla/sops/pull/435
+func LoadEncryptedFileWithBugFixes(opts GenericDecryptOpts) (*sops.Tree, error) {
+	tree, err := LoadEncryptedFile(opts.InputStore, opts.InputPath)
+	if err != nil {
+		return nil, err
+	}
+
+	encCtxBug, err := DetectKMSEncryptionContextBug(tree)
+	if err != nil {
+		return nil, err
+	}
+	if encCtxBug {
+		tree, err = FixAWSKMSEncryptionContextBug(opts, tree)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return tree, nil
+}
+
+// FixAWSKMSEncryptionContextBug is used to fix the issue described in https://github.com/mozilla/sops/pull/435
+func FixAWSKMSEncryptionContextBug(opts GenericDecryptOpts, tree *sops.Tree) (*sops.Tree, error) {
+	message := "Up until version 3.3.0 of sops there was a bug surrounding the " +
+		"use of encryption context with AWS KMS." +
+		"\nYou can read the full description of the issue here:" +
+		"\nhttps://github.com/mozilla/sops/pull/435" +
+		"\n\nIf a TTY is detected, sops will ask you if you'd like for this issue to be " +
+		"automatically fixed, which will require re-encrypting the data keys used by " +
+		"each key." +
+		"\n\nIf you are not using a TTY, sops will fix the issue for this run.\n\n"
+	fmt.Println(wordwrap.WrapString(message, 75))
+
+	persistFix := false
+
+	if terminal.IsTerminal(int(os.Stdout.Fd())) {
+		var response string
+		for response != "y" && response != "n" {
+			fmt.Println("Would you like sops to automatically fix this issue? (y/n): ")
+			_, err := fmt.Scanln(&response)
+			if err != nil {
+				return nil, err
+			}
+		}
+		if response == "n" {
+			return nil, fmt.Errorf("Exiting. User responded no.")
+		} else {
+			persistFix = true
+		}
+	}
+
+	dataKey := []byte{}
+	// If there is another key, then we should be able to just decrypt
+	// without having to try different variations of the encryption context.
+	dataKey, err := DecryptTree(DecryptTreeOpts{
+		Cipher:      opts.Cipher,
+		IgnoreMac:   opts.IgnoreMAC,
+		Tree:        tree,
+		KeyServices: opts.KeyServices,
+	})
+	if err != nil {
+		dataKey = RecoverDataKeyFromBuggyKMS(opts, tree)
+	}
+
+	if dataKey == nil {
+		return nil, NewExitError(fmt.Sprintf("Failed to decrypt, meaning there is likely another problem from the encryption context bug: %s", err), codes.ErrorDecryptingTree)
+	}
+
+	errs := tree.Metadata.UpdateMasterKeysWithKeyServices(dataKey, opts.KeyServices)
+	if len(errs) > 0 {
+		err = fmt.Errorf("Could not re-encrypt data key: %s", errs)
+		return nil, err
+	}
+
+	err = EncryptTree(EncryptTreeOpts{
+		DataKey: dataKey,
+		Tree:    tree,
+		Cipher:  opts.Cipher,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// If we are not going to persist the fix, just return the re-encrypted tree.
+	if !persistFix {
+		return tree, nil
+	}
+
+	encryptedFile, err := opts.InputStore.EmitEncryptedFile(*tree)
+	if err != nil {
+		return nil, NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree)
+	}
+
+	file, err := os.Create(opts.InputPath)
+	defer file.Close()
+	if err != nil {
+		return nil, NewExitError(fmt.Sprintf("Could not open file for writing: %s", err), codes.CouldNotWriteOutputFile)
+	}
+	_, err = file.Write(encryptedFile)
+	if err != nil {
+		return nil, err
+	}
+
+	newTree, err := LoadEncryptedFile(opts.InputStore, opts.InputPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return newTree, nil
+}
+
+// RecoverDataKeyFromBuggyKMS loops through variations on Encryption Context to
+// recover the datakey. This is used to fix the issue described in https://github.com/mozilla/sops/pull/435
+func RecoverDataKeyFromBuggyKMS(opts GenericDecryptOpts, tree *sops.Tree) []byte {
+	kgndx, kndx, originalKey := GetKMSKeyWithEncryptionCtx(tree)
+
+	keyToEdit := *originalKey
+
+	encCtxVals := map[string]interface{}{}
+	for _, v := range keyToEdit.EncryptionContext {
+		encCtxVals[*v] = ""
+	}
+
+	encCtxVariations := []map[string]*string{}
+	for ctxVal := range encCtxVals {
+		encCtxVariation := map[string]*string{}
+		for key := range keyToEdit.EncryptionContext {
+			val := ctxVal
+			encCtxVariation[key] = &val
+		}
+		encCtxVariations = append(encCtxVariations, encCtxVariation)
+	}
+
+	for _, encCtxVar := range encCtxVariations {
+		keyToEdit.EncryptionContext = encCtxVar
+		tree.Metadata.KeyGroups[kgndx][kndx] = &keyToEdit
+		dataKey, err := DecryptTree(DecryptTreeOpts{
+			Cipher:      opts.Cipher,
+			IgnoreMac:   opts.IgnoreMAC,
+			Tree:        tree,
+			KeyServices: opts.KeyServices,
+		})
+		if err == nil {
+			tree.Metadata.KeyGroups[kgndx][kndx] = originalKey
+			tree.Metadata.Version = version.Version
+			return dataKey
+		}
+	}
+
+	return nil
+}

cmd/sops/decrypt.go

@@ -20,7 +20,13 @@ type decryptOpts struct {
 }
 
 func decrypt(opts decryptOpts) (decryptedFile []byte, err error) {
-	tree, err := common.LoadEncryptedFile(opts.InputStore, opts.InputPath)
+	tree, err := common.LoadEncryptedFileWithBugFixes(common.GenericDecryptOpts{
+		Cipher:      opts.Cipher,
+		InputStore:  opts.InputStore,
+		InputPath:   opts.InputPath,
+		IgnoreMAC:   opts.IgnoreMAC,
+		KeyServices: opts.KeyServices,
+	})
 	if err != nil {
 		return nil, err
 	}

cmd/sops/edit.go

@@ -22,6 +22,7 @@ import (
 	"go.mozilla.org/sops/cmd/sops/codes"
 	"go.mozilla.org/sops/cmd/sops/common"
 	"go.mozilla.org/sops/keyservice"
+	"go.mozilla.org/sops/version"
 )
 
 type editOpts struct {
@@ -66,7 +67,7 @@ func editExample(opts editExampleOpts) ([]byte, error) {
 			KeyGroups:         opts.KeyGroups,
 			UnencryptedSuffix: opts.UnencryptedSuffix,
 			EncryptedSuffix:   opts.EncryptedSuffix,
-			Version:           version,
+			Version:           version.Version,
 			ShamirThreshold:   opts.GroupThreshold,
 		},
 		FilePath: path,
@@ -83,7 +84,13 @@ func editExample(opts editExampleOpts) ([]byte, error) {
 
 func edit(opts editOpts) ([]byte, error) {
 	// Load the file
-	tree, err := common.LoadEncryptedFile(opts.InputStore, opts.InputPath)
+	tree, err := common.LoadEncryptedFileWithBugFixes(common.GenericDecryptOpts{
+		Cipher:      opts.Cipher,
+		InputStore:  opts.InputStore,
+		InputPath:   opts.InputPath,
+		IgnoreMAC:   opts.IgnoreMAC,
+		KeyServices: opts.KeyServices,
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -201,12 +208,12 @@ func runEditorUntilOk(opts runEditorUntilOkOpts) error {
 			opts.Tree = &t
 		}
 		opts.Tree.Branches = newBranches
-		needVersionUpdated, err := AIsNewerThanB(version, opts.Tree.Metadata.Version)
+		needVersionUpdated, err := version.AIsNewerThanB(version.Version, opts.Tree.Metadata.Version)
 		if err != nil {
-			return common.NewExitError(fmt.Sprintf("Failed to compare document version %q with program version %q: %v", opts.Tree.Metadata.Version, version, err), codes.FailedToCompareVersions)
+			return common.NewExitError(fmt.Sprintf("Failed to compare document version %q with program version %q: %v", opts.Tree.Metadata.Version, version.Version, err), codes.FailedToCompareVersions)
 		}
 		if needVersionUpdated {
-			opts.Tree.Metadata.Version = version
+			opts.Tree.Metadata.Version = version.Version
 		}
 		if opts.Tree.Metadata.MasterKeyCount() == 0 {
 			log.Error("No master keys were provided, so sops can't " +

cmd/sops/encrypt.go

@@ -11,6 +11,7 @@ import (
 	"go.mozilla.org/sops/cmd/sops/codes"
 	"go.mozilla.org/sops/cmd/sops/common"
 	"go.mozilla.org/sops/keyservice"
+	"go.mozilla.org/sops/version"
 )
 
 type encryptOpts struct {
@@ -75,7 +76,7 @@ func encrypt(opts encryptOpts) (encryptedFile []byte, err error) {
 			KeyGroups:         opts.KeyGroups,
 			UnencryptedSuffix: opts.UnencryptedSuffix,
 			EncryptedSuffix:   opts.EncryptedSuffix,
-			Version:           version,
+			Version:           version.Version,
 			ShamirThreshold:   opts.GroupThreshold,
 		},
 		FilePath: path,

cmd/sops/main.go

@@ -32,6 +32,7 @@ import (
 	"go.mozilla.org/sops/stores/ini"
 	"go.mozilla.org/sops/stores/json"
 	yamlstores "go.mozilla.org/sops/stores/yaml"
+	"go.mozilla.org/sops/version"
 	"google.golang.org/grpc"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -43,7 +44,7 @@ func init() {
 }
 
 func main() {
-	cli.VersionPrinter = printVersion
+	cli.VersionPrinter = version.PrintVersion
 	app := cli.NewApp()
 
 	keyserviceFlags := []cli.Flag{
@@ -59,7 +60,7 @@ func main() {
 	app.Name = "sops"
 	app.Usage = "sops - encrypted file editor with AWS KMS, GCP KMS, Azure Key Vault and GPG support"
 	app.ArgsUsage = "sops [options] file"
-	app.Version = version
+	app.Version = version.Version
 	app.Authors = []cli.Author{
 		{Name: "Julien Vehent", Email: "[email protected]"},
 		{Name: "Adrian Utrilla", Email: "[email protected]"},

cmd/sops/rotate.go

@@ -23,7 +23,13 @@ type rotateOpts struct {
 }
 
 func rotate(opts rotateOpts) ([]byte, error) {
-	tree, err := common.LoadEncryptedFile(opts.InputStore, opts.InputPath)
+	tree, err := common.LoadEncryptedFileWithBugFixes(common.GenericDecryptOpts{
+		Cipher:      opts.Cipher,
+		InputStore:  opts.InputStore,
+		InputPath:   opts.InputPath,
+		IgnoreMAC:   opts.IgnoreMAC,
+		KeyServices: opts.KeyServices,
+	})
 	if err != nil {
 		return nil, err
 	}