Draft: Fine-grained excludes

acceptance.bats

@@ -74,7 +74,7 @@
 @test "Test command works with nested namespaces" {
   run ./conftest test --namespace main.gke -p examples/hcl1/policy/ examples/hcl1/gke.tf --no-color
   [ "$status" -eq 1 ]
-  [ "${lines[1]}" = "1 test, 0 passed, 0 warnings, 1 failure, 0 exceptions" ]
+  [ "${lines[1]}" = "1 test, 0 passed, 0 warnings, 1 failure, 0 exceptions, 0 exclusions" ]
 }
 
 @test "Verify command has trace flag" {
@@ -347,13 +347,13 @@
 @test "The number of tests run is accurate" {
   run ./conftest test -p examples/kubernetes/policy examples/kubernetes/service.yaml --no-color
   [ "$status" -eq 0 ]
-  [ "${lines[1]}" = "5 tests, 4 passed, 1 warning, 0 failures, 0 exceptions" ]
+  [ "${lines[1]}" = "5 tests, 4 passed, 1 warning, 0 failures, 0 exceptions, 0 exclusions" ]
 }
 
 @test "Exceptions reported correctly" {
   run ./conftest test -p examples/exceptions/policy examples/exceptions/deployments.yaml --no-color
   [ "$status" -eq 1 ]
-  [ "${lines[2]}" = "2 tests, 0 passed, 0 warnings, 1 failure, 1 exception" ]
+  [ "${lines[2]}" = "2 tests, 0 passed, 0 warnings, 1 failure, 1 exception, 0 exclusions" ]
 }
 
 @test "Exceptions output" {
@@ -365,7 +365,7 @@
 @test "Suppress exceptions output" {
   run ./conftest test -p examples/exceptions/policy examples/exceptions/deployments.yaml --no-color --suppress-exceptions
   [ "$status" -eq 1 ]
-  [ "${lines[1]}" = "2 tests, 0 passed, 0 warnings, 1 failure, 1 exception" ]
+  [ "${lines[1]}" = "2 tests, 0 passed, 0 warnings, 1 failure, 1 exception, 0 exclusions" ]
 }
 
 @test "Can combine yaml files" {

examples/excludes/main.tf

@@ -0,0 +1,8 @@
+
+
+
+resource "null_resource" "exception-name" {}
+
+resource "null_resource" "invalid-name" {}
+
+resource "invalid_type" "valid_name" {}

examples/excludes/policy/deny.rego

@@ -0,0 +1,28 @@
+package main
+
+exceptions = {"exception-name"}
+
+deny_name[result] {
+	input.resource[_][name]
+	contains(name, "-")
+	msg := sprintf("Resource Name '%s' contains dashes", [name])
+	result := {
+		"msg": msg,
+		"resource-name": name,
+	}
+}
+
+deny_resource_type[msg] {
+	input.resource[type]
+	type == "invalid_type"
+	msg := sprintf("Resource Type '%s' is invalid", [type])
+}
+
+exclude_name[attrs] {
+	exceptions[name]
+	attrs := [{"resource-name": name}]
+}
+
+exception[rules] {
+	rules := ["resource_type"]
+}

examples/excludes2/deployment.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mydep
+spec:
+  template:
+    spec:
+      containers:
+        - name: web
+          image: nginx
+          ports:
+            - containerPort: 8080
+          securityContext:
+            runAsNonRoot: false
+        - name: host-agent
+          image: host-agent
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: not-mydep
+spec:
+  template:
+    spec:
+      containers:
+        - name: web
+          image: nginx
+          ports:
+            - containerPort: 8080
+          securityContext:
+            runAsNonRoot: true
+        - name: host-agent
+          image: host-agent
+          securityContext:
+            runAsNonRoot: true

examples/excludes2/policy/deny.rego

@@ -0,0 +1,29 @@
+package main
+
+deny_root[result] {
+	input.kind == "Deployment"
+	c = input.spec.template.spec.containers[_]
+	not c.securityContext.runAsNonRoot
+
+	# key "msg" is required to be set.
+	result := {
+		"container": c.name,
+		"deployment": input.metadata.name,
+		"msg": sprintf("container %s in deployment %s doesn't set runAsNonRoot", [c.name, input.metadata.name]),
+	}
+}
+
+root_exceptions = [{"deployment": "mydep", "containers": ["host-agent"]}]
+
+# Here the exception I want to be able to express is "mydep can run host-agent as root".
+# But not web as root
+exclude_root[attrs] {
+	deployment := input.metadata.name
+	container := input.spec.template.spec.containers[_].name
+	exception := root_exceptions[_]
+
+	deployment == exception.deployment
+	container == exception.containers[_]
+
+	attrs = [{"container": container, "deployment": deployment}]
+}

output/result.go

@@ -82,6 +82,7 @@ type CheckResult struct {
 	Warnings   []Result      `json:"warnings,omitempty"`
 	Failures   []Result      `json:"failures,omitempty"`
 	Exceptions []Result      `json:"exceptions,omitempty"`
+	Excludes   []Result      `json:"excludes,omitempty"`
 	Queries    []QueryResult `json:"queries,omitempty"`
 }
 

output/standard.go

@@ -57,6 +57,7 @@ func (s *Standard) Output(results []CheckResult) error {
 
 	var totalFailures int
 	var totalExceptions int
+	var totalExclusions int
 	var totalWarnings int
 	var totalSuccesses int
 	var totalSkipped int
@@ -95,14 +96,19 @@ func (s *Standard) Output(results []CheckResult) error {
 			}
 		}
 
+		for _, exclude := range result.Excludes {
+			fmt.Fprintln(s.Writer, colorizer.Colorize("EXCL", aurora.BlueFg), indicator, namespace, exclude.Message)
+		}
+
 		totalFailures += len(result.Failures)
 		totalExceptions += len(result.Exceptions)
 		totalWarnings += len(result.Warnings)
+		totalExclusions += len(result.Excludes)
 		totalSkipped += len(result.Skipped)
 		totalSuccesses += result.Successes
 	}
 
-	totalTests := totalFailures + totalExceptions + totalWarnings + totalSuccesses + totalSkipped
+	totalTests := totalFailures + totalExceptions + totalWarnings + totalSuccesses + totalExclusions + totalSkipped
 
 	var pluralSuffixTests string
 	if totalTests != 1 {
@@ -124,12 +130,18 @@ func (s *Standard) Output(results []CheckResult) error {
 		pluralSuffixExceptions = "s"
 	}
 
-	outputText := fmt.Sprintf("%v test%s, %v passed, %v warning%s, %v failure%s, %v exception%s",
+	var pluralSuffixExclusions string
+	if totalExclusions != 1 {
+		pluralSuffixExclusions = "s"
+	}
+
+	outputText := fmt.Sprintf("%v test%s, %v passed, %v warning%s, %v failure%s, %v exception%s, %v exclusion%s",
 		totalTests, pluralSuffixTests,
 		totalSuccesses,
 		totalWarnings, pluralSuffixWarnings,
 		totalFailures, pluralSuffixFailures,
 		totalExceptions, pluralSuffixExceptions,
+		totalExclusions, pluralSuffixExclusions,
 	)
 
 	if s.ShowSkipped {

output/standard_test.go

@@ -28,7 +28,7 @@ func TestStandard(t *testing.T) {
 				"WARN - foo.yaml - namespace - first warning",
 				"FAIL - foo.yaml - namespace - first failure",
 				"",
-				"2 tests, 0 passed, 1 warning, 1 failure, 0 exceptions",
+				"2 tests, 0 passed, 1 warning, 1 failure, 0 exceptions, 0 exclusions",
 				"",
 			},
 		},
@@ -46,7 +46,7 @@ func TestStandard(t *testing.T) {
 				"WARN - - namespace - first warning",
 				"FAIL - - namespace - first failure",
 				"",
-				"2 tests, 0 passed, 1 warning, 1 failure, 0 exceptions",
+				"2 tests, 0 passed, 1 warning, 1 failure, 0 exceptions, 0 exclusions",
 				"",
 			},
 		},
@@ -66,7 +66,7 @@ func TestStandard(t *testing.T) {
 				"WARN - foo.yaml - namespace - first warning",
 				"FAIL - foo.yaml - namespace - first failure",
 				"",
-				"3 tests, 0 passed, 1 warning, 1 failure, 0 exceptions, 1 skipped",
+				"3 tests, 0 passed, 1 warning, 1 failure, 0 exceptions, 0 exclusions, 1 skipped",
 				"",
 			},
 		},
@@ -85,7 +85,7 @@ func TestStandard(t *testing.T) {
 				"WARN - - namespace - first warning",
 				"FAIL - - namespace - first failure",
 				"",
-				"2 tests, 0 passed, 1 warning, 1 failure, 0 exceptions, 0 skipped",
+				"2 tests, 0 passed, 1 warning, 1 failure, 0 exceptions, 0 exclusions, 0 skipped",
 				"",
 			},
 		},

policy/engine.go

@@ -3,6 +3,7 @@ package policy
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -141,6 +142,7 @@ func (e *Engine) Check(ctx context.Context, configs map[string]interface{}, name
 				checkResult.Failures = append(checkResult.Failures, result.Failures...)
 				checkResult.Warnings = append(checkResult.Warnings, result.Warnings...)
 				checkResult.Exceptions = append(checkResult.Exceptions, result.Exceptions...)
+				checkResult.Excludes = append(checkResult.Excludes, result.Excludes...)
 				checkResult.Queries = append(checkResult.Queries, result.Queries...)
 			}
 			checkResults = append(checkResults, checkResult)
@@ -241,6 +243,7 @@ func (e *Engine) check(ctx context.Context, path string, config interface{}, nam
 
 	var rules []string
 	var ruleCount int
+
 	for _, module := range e.Modules() {
 		currentNamespace := strings.Replace(module.Package.Path.String(), "data.", "", 1)
 		if currentNamespace != namespace {
@@ -306,6 +309,7 @@ func (e *Engine) check(ctx context.Context, path string, config interface{}, nam
 
 		var failures []output.Result
 		var warnings []output.Result
+		var excludes []output.Result
 		for _, ruleResult := range ruleQueryResult.Results {
 
 			// Exceptions have already been accounted for in the exception query so
@@ -319,6 +323,30 @@ func (e *Engine) check(ctx context.Context, path string, config interface{}, nam
 				continue
 			}
 
+			result, err := json.Marshal(ruleResult.Metadata)
+			if err != nil {
+				return output.CheckResult{}, fmt.Errorf("json marshal: %w", err)
+			}
+
+			// If we have a non-null metadata response, then we are eligible to exclude the policy.
+			// Otherwise we can just skip & process the policy violation
+			if string(result) != "null" {
+				localExcludeQuery := fmt.Sprintf("data.%s.exclude_%s[_][_] = %s", namespace, removeRulePrefix(rule), result)
+				localExcludeQueryResult, err := e.query(ctx, config, localExcludeQuery)
+				if err != nil {
+					return output.CheckResult{}, fmt.Errorf("query exception: %w", err)
+				}
+
+				// If the query was a failure, let's have a look & see if an exception was written for it.
+				if len(localExcludeQueryResult.Results) > 0 {
+					// append an exception & continue
+					localExcludeResult := localExcludeQueryResult.Results[0]
+					localExcludeResult.Message = localExcludeQuery
+					excludes = append(excludes, localExcludeResult)
+					continue
+				}
+
+			}
 			if isFailure(rule) {
 				failures = append(failures, ruleResult)
 			} else {
@@ -329,6 +357,7 @@ func (e *Engine) check(ctx context.Context, path string, config interface{}, nam
 		checkResult.Failures = append(checkResult.Failures, failures...)
 		checkResult.Warnings = append(checkResult.Warnings, warnings...)
 		checkResult.Exceptions = append(checkResult.Exceptions, exceptions...)
+		checkResult.Excludes = append(checkResult.Excludes, excludes...)
 
 		checkResult.Queries = append(checkResult.Queries, exceptionQueryResult, ruleQueryResult)
 	}