Merge pull request #693 from mozilla/develop

v3.6.0 RC (develop -> master)

Author: ajvb

.circleci/config.yml

@@ -1,9 +1,21 @@
-version: 2
+version: 2.1
+
+workflows:
+  build-and-deploy:
+    jobs:
+      - build
+      - push:     
+          filters:
+            tags:
+              only: /^v.*/   
+            branches:
+              ignore: /.*/
 jobs:
   build:
     working_directory: /go/src/go.mozilla.org/sops
     docker:
       - image: circleci/golang:1.13
+    resource_class: large
     steps:
       - checkout
       - setup_remote_docker
@@ -12,10 +24,79 @@ jobs:
           command: |
             docker build -t mozilla/sops .
             docker tag mozilla/sops "mozilla/sops:$CIRCLE_SHA1"
+
+  push:
+    machine: true
+    resource_class: large
+    steps:
+      - checkout
+      - run: 
+          name: semver check
+          command: |
+            MAJOR=$(echo ${CIRCLE_TAG#v} | cut -d"." -f1)
+            MINOR=$(echo ${CIRCLE_TAG#v} | cut -d"." -f2)
+            PATCH=$(echo ${CIRCLE_TAG#v} | cut -d"." -f3)
+            echo "export MAJOR=${MAJOR}" >> $BASH_ENV
+            echo "export MINOR=${MINOR}" >> $BASH_ENV
+            echo "export PATCH=${PATCH}" >> $BASH_ENV
+
+            if [ -z $MAJOR ];then
+            cat \<< EOF
+            Failure Info:
+
+            This job uses the semver from the git TAG as the public version to publish.
+
+            - This should only run on workflows triggered by a tag. 
+            - The tag name should be a semver like 'v1.2.3' 
+            - The version should follow conventions documented at https://github.com/fsaintjacques/semver-tool
+            EOF
+            exit 1
+            fi
+      - run:
+          name: Build containers
+          command: |
+            docker build -t mozilla/sops .
+            docker build -f Dockerfile.alpine -t mozilla/sops:alpine .
       - run:
-          name: Push containers
+          name: Tag & Push containers
           command: |
-            if [ "${CIRCLE_BRANCH}" == "master" ]; then
-                ${GOPATH}/src/go.mozilla.org/sops/bin/ci/deploy_dockerhub.sh "latest"
-                ${GOPATH}/src/go.mozilla.org/sops/bin/ci/deploy_dockerhub.sh "$CIRCLE_SHA1"
+            #latest
+            bin/ci/deploy_dockerhub.sh "latest"
+            bin/ci/deploy_dockerhub.sh "alpine"
+
+            # by sha
+            echo "Tag and push mozilla/sops:$CIRCLE_SHA1"
+            docker tag mozilla/sops "mozilla/sops:$CIRCLE_SHA1"
+            bin/ci/deploy_dockerhub.sh "$CIRCLE_SHA1"
+
+            # no sha for alpine
+
+            # by semver
+            # v1.2.3
+            if [ ! -z $PATCH ];then
+              echo "Tag and Push mozilla/sops:v$MAJOR.$MINOR.$PATCH"
+              docker tag mozilla/sops "mozilla/sops:v$MAJOR.$MINOR.$PATCH"
+              bin/ci/deploy_dockerhub.sh "v$MAJOR.$MINOR.$PATCH"
+
+              echo "Tag and Push mozilla/sops:v$MAJOR.$MINOR.$PATCH-alpine"
+              docker tag mozilla/sops:alpine "mozilla/sops:v$MAJOR.$MINOR.$PATCH-alpine"
+              bin/ci/deploy_dockerhub.sh "v$MAJOR.$MINOR.$PATCH-alpine"
+            fi
+            # v1.2
+            if [ ! -z $MINOR ];then
+              echo "Tag and Push mozilla/sops:v$MAJOR.$MINOR"
+              docker tag mozilla/sops "mozilla/sops:v$MAJOR.$MINOR"
+              bin/ci/deploy_dockerhub.sh "v$MAJOR.$MINOR"
+
+              echo "Tag and Push mozilla/sops:v$MAJOR.$MINOR-alpine"
+              docker tag mozilla/sops:alpine "mozilla/sops:v$MAJOR.$MINOR-alpine"
+              bin/ci/deploy_dockerhub.sh "v$MAJOR.$MINOR-alpine"
             fi
+            # v1
+            echo "Tag and Push mozilla/sops:v$MAJOR"
+            docker tag mozilla/sops "mozilla/sops:v$MAJOR"
+            bin/ci/deploy_dockerhub.sh "v$MAJOR"
+
+            echo "Tag and Push mozilla/sops:v$MAJOR-alpine"
+            docker tag mozilla/sops:alpine "mozilla/sops:v$MAJOR-alpine"
+            bin/ci/deploy_dockerhub.sh "v$MAJOR-alpine"

.dockerignore

@@ -0,0 +1,3 @@
+/.git
+/Dockerfile
+/Dockerfile.alpine

.gitignore

@@ -1,3 +1,5 @@
 target
 Cargo.lock
 vendor/
+coverage.txt
+profile.out

.sops.yaml

@@ -1,2 +1,4 @@
 creation_rules:
-    - pgp: 1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A
+    - pgp: >-
+        FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4,
+        D7229043384BCC60326C6FB9D8720D957C3D3074

CHANGELOG.rst

@@ -1,6 +1,37 @@
 Changelog
 =========
 
+3.6.0
+-----
+Features:
+
+    * Support for encrypting data through the use of Hashicorp Vault (#655)
+    * `sops publish` now supports `--recursive` flag for publishing all files in a directory (#602)
+    * `sops publish` now supports `--omit-extensions` flag for omitting the extension in the destination path (#602)
+    * sops now supports JSON arrays of arrays (#642)
+
+Improvements:
+
+    * Updates and standardization for the dotenv store (#612, #622)
+    * Close temp files after using them for edit command (#685)
+
+Bug fixes:
+
+    * AWS SDK usage now correctly resolves the `~/.aws/config` file (#680)
+    * `sops updatekeys` now correctly matches config rules (#682)
+    * `sops updatekeys` now correctly uses the config path cli flag (#672)
+    * Partially empty sops config files don't break the use of sops anymore (#662)
+    * Fix possible infinite loop in PGP's passphrase prompt call (#690)
+
+Project changes:
+
+    * Dockerfile now based off of golang version 1.14 (#649)
+    * Push alpine version of docker image to Dockerhub (#609)
+    * Push major, major.minor, and major.minor.patch tagged docker images to Dockerhub (#607)
+    * Removed out of date contact information (#668)
+    * Update authors in the cli help text (#645)
+
+
 3.5.0
 -----
 Features:

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.12
+FROM golang:1.14
 
 COPY . /go/src/go.mozilla.org/sops
 WORKDIR /go/src/go.mozilla.org/sops

Dockerfile.alpine

@@ -0,0 +1,17 @@
+FROM golang:1.12-alpine3.10 AS builder
+
+RUN apk --no-cache add make
+
+COPY . /go/src/go.mozilla.org/sops
+WORKDIR /go/src/go.mozilla.org/sops
+
+RUN CGO_ENABLED=1 make install
+
+
+FROM alpine:3.10
+
+RUN apk --no-cache add \
+  vim ca-certificates
+ENV EDITOR vim
+COPY --from=builder /go/bin/sops /usr/local/bin/sops
+ENTRYPOINT ["/usr/local/bin/sops"]

README.rst

@@ -48,9 +48,6 @@ Or whatever variation of the above fits your system and shell.
 
 To use **sops** as a library, take a look at the `decrypt package <https://godoc.org/go.mozilla.org/sops/decrypt>`_.
 
-**Questions?** ping "ulfr" and "autrilla" in ``#security`` on `irc.mozilla.org <https://wiki.mozilla.org/IRC>`_
-(use a web client like `mibbit <https://chat.mibbit.com>`_ ).
-
 **What happened to Python Sops?** We rewrote Sops in Go to solve a number of
 deployment issues, but the Python branch still exists under ``python-sops``. We
 will keep maintaining it for a while, and you can still ``pip install sops``,
@@ -290,6 +287,66 @@ And decrypt it using::
 	 $ sops --decrypt test.enc.yaml
 
 
+Encrypting using Hashicorp Vault
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We assume you have an instance (or more) of Vault running and you have privileged access to it. For instructions on how to deploy a secure instance of Vault, refer to Hashicorp's official documentation.
+
+To easily deploy Vault locally: (DO NOT DO THIS FOR PRODUCTION!!!) 
+
+.. code:: bash
+
+	$ docker run -d -p8200:8200 vault:1.2.0 server -dev -dev-root-token-id=toor
+
+
+.. code:: bash
+
+	$ # Substitute this with the address Vault is running on
+	$ export VAULT_ADDR=http://127.0.0.1:8200 
+
+	$ # this may not be necessary in case you previously used `vault login` for production use
+	$ export VAULT_TOKEN=toor 
+	
+	$ # to check if Vault started and is configured correctly
+	$ vault status
+	Key             Value
+	---             -----
+	Seal Type       shamir
+	Initialized     true
+	Sealed          false
+	Total Shares    1
+	Threshold       1
+	Version         1.2.0
+	Cluster Name    vault-cluster-618cc902
+	Cluster ID      e532e461-e8f0-1352-8a41-fc7c11096908
+	HA Enabled      false
+
+	$ # It is required to enable a transit engine if not already done (It is suggested to create a transit engine specifically for sops, in which it is possible to have multiple keys with various permission levels)
+	$ vault secrets enable -path=sops transit
+	Success! Enabled the transit secrets engine at: sops/
+
+	$ # Then create one or more keys
+	$ vault write sops/keys/firstkey type=rsa-4096
+	Success! Data written to: sops/keys/firstkey
+
+	$ vault write sops/keys/secondkey type=rsa-2048
+	Success! Data written to: sops/keys/secondkey
+
+	$ vault write sops/keys/thirdkey type=chacha20-poly1305
+	Success! Data written to: sops/keys/thirdkey
+
+	$ sops --hc-vault-transit $VAULT_ADDR/v1/sops/keys/firstkey vault_example.yml
+
+	$ cat <<EOF > .sops.yaml
+	creation_rules:
+		- path_regex: \.dev\.yaml$
+		  hc_vault_transit_uri: "$VAULT_ADDR/v1/sops/keys/secondkey"
+		- path_regex: \.prod\.yaml$
+		  hc_vault_transit_uri: "$VAULT_ADDR/v1/sops/keys/thirdkey"
+	EOF
+
+	$ sops --verbose -e prod/raw.yaml > prod/encrypted.yaml
+
 Adding and removing keys
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -546,6 +603,7 @@ can manage the three sets of configurations for the three types of files:
 		- path_regex: \.prod\.yaml$
 		  kms: 'arn:aws:kms:us-west-2:361527076523:key/5052f06a-5d3f-489e-b86c-57201e06f31e+arn:aws:iam::361527076523:role/hiera-sops-prod,arn:aws:kms:eu-central-1:361527076523:key/cb1fab90-8d17-42a1-a9d8-334968904f94+arn:aws:iam::361527076523:role/hiera-sops-prod'
 		  pgp: 'FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4'
+		  hc_vault_uris: "http://localhost:8200/v1/sops/keys/thirdkey"
 
 		# gcp files using GCP KMS
 		- path_regex: \.gcp\.yaml$
@@ -865,21 +923,21 @@ written to disk.
            "AWS_ACCESS_KEY_ID": "AKIAIOSFODNN7EXAMPLE",
            "AWS_SECRET_KEY": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
    }
-   
+
    # decrypt out.json and run a command
    # the command prints the environment variable and runs a script that uses it
    $ sops exec-env out.json 'echo secret: $database_password; ./database-import'
    secret: jf48t9wfw094gf4nhdf023r
-   
+
    # launch a shell with the secrets available in its environment
    $ sops exec-env out.json 'sh'
    sh-3.2# echo $database_password
    jf48t9wfw094gf4nhdf023r
-   
+
    # the secret is not accessible anywhere else
    sh-3.2$ exit
    $ echo your password: $database_password
-   your password: 
+   your password:
 
 
 If the command you want to run only operates on files, you can use ``exec-file``
@@ -904,7 +962,7 @@ substituted with the temporary file path (whether a FIFO or an actual file).
            "AWS_ACCESS_KEY_ID": "AKIAIOSFODNN7EXAMPLE",
            "AWS_SECRET_KEY": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
    }
-   
+
    # launch a shell with a variable TMPFILE pointing to the temporary file
    $ sops exec-file --no-fifo out.json 'TMPFILE={} sh'
    sh-3.2$ echo $TMPFILE
@@ -934,7 +992,7 @@ for added security.
    # the encrypted file can't be read by the current user
    $ cat out.json
    cat: out.json: Permission denied
-   
+
    # execute sops as root, decrypt secrets, then drop privileges
    $ sudo sops exec-env --user nobody out.json 'sh'
    sh-3.2$ echo $database_password
@@ -968,6 +1026,7 @@ This command requires a ``.sops.yaml`` configuration file. Below is an example:
         vault_kv_mount_name: "secret/" # default
         vault_kv_version: 2 # default
         path_regex: vault/*
+        omit_extensions: true
 
 The above configuration will place all files under ``s3/*`` into the S3 bucket ``sops-secrets``,
 all files under ``gcs/*`` into the GCS bucket ``sops-secrets``, and the contents of all files under
@@ -977,6 +1036,11 @@ published to S3 and GCS, it will decrypt them and re-encrypt them using the
 
 You would deploy a file to S3 with a command like: ``sops publish s3/app.yaml``
 
+To publish all files in selected directory recursively, you need to specify ``--recursive`` flag.
+
+If you don't want file extension to appear in destination secret path, use ``--omit-extensions``
+flag or ``omit_extensions: true`` in the destination rule in ``.sops.yaml``.
+
 Publishing to Vault
 *******************
 
@@ -991,6 +1055,9 @@ configuring the client.
 ``vault_kv_mount_name`` is used if your Vault KV is mounted somewhere other than ``secret/``.
 ``vault_kv_version`` supports ``1`` and ``2``, with ``2`` being the default.
 
+If destination secret path already exists in Vault and contains same data as the source file, it
+will be skipped.
+
 Below is an example of publishing to Vault (using token auth with a local dev instance of Vault).
 
 .. code:: bash
@@ -1293,9 +1360,10 @@ You can import sops as a module and use it in your python program.
 	tree = sops.walk_and_decrypt(tree, sops_key)
 	sops.write_file(tree, path=path, filetype=pathtype)
 
-Note: this uses the previous implemenation of `sops` written in python,
+Note: this uses the previous implementation of `sops` written in python,
+
 and so doesn't support newer features such as GCP-KMS.
-To use the current version, call out to `sops` using `subprocess.check_output`
+To use the current version, call out to ``sops`` using ``subprocess.run``
 
 Showing diffs in cleartext in git
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

cmd/sops/edit.go

@@ -132,6 +132,9 @@ func editTree(opts editOpts, tree *sops.Tree, dataKey []byte) ([]byte, error) {
 	if err != nil {
 		return nil, common.NewExitError(fmt.Sprintf("Could not write output file: %s", err), codes.CouldNotWriteOutputFile)
 	}
+	
+	// Close temporary file, since Windows won't delete the file unless it's closed beforehand
+	defer tmpfile.Close()
 
 	// Compute file hash to detect if the file has been edited
 	origHash, err := hashFile(tmpfile.Name())