From 8c519163d605d5a6a6a6c56ae036485e7e20c462 Mon Sep 17 00:00:00 2001 From: John McCabe Date: Fri, 15 Sep 2017 15:03:57 +0100 Subject: [PATCH 1/5] (MAINT) fix docker capability schema url --- capabilities/docker/docker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capabilities/docker/docker.go b/capabilities/docker/docker.go index 9ace24c..adde9df 100644 --- a/capabilities/docker/docker.go +++ b/capabilities/docker/docker.go @@ -14,7 +14,7 @@ var dockerDescription = `The 'docker' capability captures information related to var dockerCapability = dockeradapter.DockerAPICapability{ Capability: types.Capability{ - Schema: "http://puppet.com/lumogon/capability/label/draft-01/schema#1", + Schema: "http://puppet.com/lumogon/capability/docker/draft-01/schema#1", Title: "Docker Server Information", Name: "docker", Description: dockerDescription, From 626fae978122f85fd738645cc84c48cf21494ce7 Mon Sep 17 00:00:00 2001 From: John McCabe Date: Wed, 20 Sep 2017 15:01:15 +0100 Subject: [PATCH 2/5] (DI-489) Add basic container info capability This commit adds a basic container info capability that harvests a subset of the data available to ContainerInspect, it does not currently expose any structured data (pending UI updates) or any data that may container sensitive data. Have tested against both Docker 1.12.6 (API 1.24) and 17.06.2-ce (API 1.30). --- capabilities/container/container.go | 113 ++++++++++++++++++++++++++++ capabilities/container/init.go | 7 ++ capabilities/init.go | 2 + 3 files changed, 122 insertions(+) create mode 100644 capabilities/container/container.go create mode 100644 capabilities/container/init.go diff --git a/capabilities/container/container.go b/capabilities/container/container.go new file mode 100644 index 0000000..8f4f422 --- /dev/null +++ b/capabilities/container/container.go @@ -0,0 +1,113 @@ +package container + +import ( + "context" + "fmt" + "strings" + + "github.com/docker/go-connections/nat" + "github.com/puppetlabs/lumogon/capabilities/payloadfilter" + "github.com/puppetlabs/lumogon/capabilities/registry" + "github.com/puppetlabs/lumogon/dockeradapter" + "github.com/puppetlabs/lumogon/logging" + "github.com/puppetlabs/lumogon/types" +) + +var containerDescription = `The 'container' capability captures detailed container information` + +var containerCapability = dockeradapter.DockerAPICapability{ + Capability: types.Capability{ + Schema: "http://puppet.com/lumogon/capability/container/draft-01/schema#1", + Title: "Container Information", + Name: "container", + Description: containerDescription, + Type: "dockerapi", + Payload: nil, + SupportedOS: map[string]int{"all": 1}, + }, + Harvest: func(capability *dockeradapter.DockerAPICapability, client dockeradapter.Harvester, id string, target types.TargetContainer) { + logging.Debug("[Container Info] Harvesting container information associated with %s [%s]", target.Name, target.ID) + capability.HarvestID = id + + version, err := InspectContainer(client, target.ID) + if err != nil { + capability.PayloadError(err.Error()) + return + } + + filtered, _ := payloadfilter.Filter(version) + + capability.Payload = filtered + }, +} + +// InspectContainer Extracts and returns a formatted map[string]interface{} containing +// a subset of information returned by ContainerInspect +func InspectContainer(client dockeradapter.Harvester, targetID string) (map[string]interface{}, error) { + ctx := context.Background() + c, err := client.ContainerInspect(ctx, targetID) + if err != nil { + return nil, err + } + + // TODO - **IMPORTANT** this contains only a subset of the information available + // it explicitly avoids including any structured data (ports/mappings etc) pending + // support in the UI, it also avoids any config that could potentially contain + // sensitive data. + + result := map[string]interface{}{ + "Hostname": c.Config.Hostname, + "Domainname": c.Config.Domainname, + "User": c.Config.User, + "Image": c.Config.Image, + "AttachStdin": fmt.Sprintf("%t", c.Config.AttachStdin), + "AttachStdout": fmt.Sprintf("%t", c.Config.AttachStdout), + "AttachStderr": fmt.Sprintf("%t", c.Config.AttachStderr), + "Tty": fmt.Sprintf("%t", c.Config.Tty), + "OpenStdin": fmt.Sprintf("%t", c.Config.OpenStdin), + "StdinOnce": fmt.Sprintf("%t", c.Config.StdinOnce), + "Privileged": fmt.Sprintf("%t", c.HostConfig.Privileged), + "PublishAllPorts": fmt.Sprintf("%t", c.HostConfig.PublishAllPorts), + "ReadonlyRootfs": fmt.Sprintf("%t", c.HostConfig.ReadonlyRootfs), + "ShmSize": fmt.Sprintf("%d", c.HostConfig.ShmSize), + "CapAdd": strings.Join(c.HostConfig.CapAdd, ", "), + "CapDrop": strings.Join(c.HostConfig.CapDrop, ", "), + "Runtime": c.HostConfig.Runtime, + "CPUShares": fmt.Sprintf("%d", c.HostConfig.Resources.CPUShares), + "Memory": fmt.Sprintf("%d", c.HostConfig.Resources.Memory), + "NanoCPUs": fmt.Sprintf("%d", c.HostConfig.Resources.NanoCPUs), + "CPUPeriod": fmt.Sprintf("%d", c.HostConfig.Resources.CPUPeriod), + "CPUQuota": fmt.Sprintf("%d", c.HostConfig.Resources.CPUQuota), + "CPURealtimePeriod": fmt.Sprintf("%d", c.HostConfig.Resources.CPURealtimePeriod), + "CPURealtimeRuntime": fmt.Sprintf("%d", c.HostConfig.Resources.CPURealtimeRuntime), + "CpusetCpus": c.HostConfig.Resources.CpusetCpus, + "CpusetMems": c.HostConfig.Resources.CpusetMems, + "DiskQuota": fmt.Sprintf("%d", c.HostConfig.Resources.DiskQuota), + "KernelMemory": fmt.Sprintf("%d", c.HostConfig.Resources.KernelMemory), + "MemoryReservation": fmt.Sprintf("%d", c.HostConfig.Resources.MemoryReservation), + "MemorySwap": fmt.Sprintf("%d", c.HostConfig.Resources.MemorySwap), + "MemorySwappiness": fmt.Sprintf("%d", *c.HostConfig.Resources.MemorySwappiness), + "OomKillDisable": fmt.Sprintf("%t", *c.HostConfig.Resources.OomKillDisable), + "PidsLimit": fmt.Sprintf("%d", c.HostConfig.Resources.PidsLimit), + } + + logging.Debug("[Container Info] Harvested [%+v]", result) + + return result, nil +} + +func ports(m nat.PortSet) []string { + keys := make([]string, len(m)) + + i := 0 + for k := range m { + keys[i] = k.Port() + i++ + } + return keys +} + +func init() { + logging.Debug("[Container Info] Initialising capability: %s", containerCapability.Title) + registry.Registry.Add(containerCapability) +} diff --git a/capabilities/container/init.go b/capabilities/container/init.go new file mode 100644 index 0000000..750192f --- /dev/null +++ b/capabilities/container/init.go @@ -0,0 +1,7 @@ +package container + +// Init exists to allow container init() functions to run when +// invoked from the capabilities Init function, which is +// itself invoked by the Lumogon command handler. +func Init() { +} diff --git a/capabilities/init.go b/capabilities/init.go index 3af9986..d4c5437 100644 --- a/capabilities/init.go +++ b/capabilities/init.go @@ -1,6 +1,7 @@ package capabilities import ( + "github.com/puppetlabs/lumogon/capabilities/container" "github.com/puppetlabs/lumogon/capabilities/diff" "github.com/puppetlabs/lumogon/capabilities/docker" "github.com/puppetlabs/lumogon/capabilities/host" @@ -16,4 +17,5 @@ func Init() { label.Init() ospackages.Init() diff.Init() + container.Init() } From 93db6c679a7212d1c924c9f8fe0f2b4a67e3a161 Mon Sep 17 00:00:00 2001 From: John McCabe Date: Thu, 21 Sep 2017 12:04:05 +0100 Subject: [PATCH 3/5] (DI-489) Lowercase keys for container capability --- capabilities/container/container.go | 72 ++++++++++++++--------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/capabilities/container/container.go b/capabilities/container/container.go index 8f4f422..0cef7de 100644 --- a/capabilities/container/container.go +++ b/capabilities/container/container.go @@ -26,7 +26,7 @@ var containerCapability = dockeradapter.DockerAPICapability{ SupportedOS: map[string]int{"all": 1}, }, Harvest: func(capability *dockeradapter.DockerAPICapability, client dockeradapter.Harvester, id string, target types.TargetContainer) { - logging.Debug("[Container Info] Harvesting container information associated with %s [%s]", target.Name, target.ID) + logging.Debug("[Container Info] Harvesting container information associated with %s [%s]\n", target.Name, target.ID) capability.HarvestID = id version, err := InspectContainer(client, target.ID) @@ -56,42 +56,42 @@ func InspectContainer(client dockeradapter.Harvester, targetID string) (map[stri // sensitive data. result := map[string]interface{}{ - "Hostname": c.Config.Hostname, - "Domainname": c.Config.Domainname, - "User": c.Config.User, - "Image": c.Config.Image, - "AttachStdin": fmt.Sprintf("%t", c.Config.AttachStdin), - "AttachStdout": fmt.Sprintf("%t", c.Config.AttachStdout), - "AttachStderr": fmt.Sprintf("%t", c.Config.AttachStderr), - "Tty": fmt.Sprintf("%t", c.Config.Tty), - "OpenStdin": fmt.Sprintf("%t", c.Config.OpenStdin), - "StdinOnce": fmt.Sprintf("%t", c.Config.StdinOnce), - "Privileged": fmt.Sprintf("%t", c.HostConfig.Privileged), - "PublishAllPorts": fmt.Sprintf("%t", c.HostConfig.PublishAllPorts), - "ReadonlyRootfs": fmt.Sprintf("%t", c.HostConfig.ReadonlyRootfs), - "ShmSize": fmt.Sprintf("%d", c.HostConfig.ShmSize), - "CapAdd": strings.Join(c.HostConfig.CapAdd, ", "), - "CapDrop": strings.Join(c.HostConfig.CapDrop, ", "), - "Runtime": c.HostConfig.Runtime, - "CPUShares": fmt.Sprintf("%d", c.HostConfig.Resources.CPUShares), - "Memory": fmt.Sprintf("%d", c.HostConfig.Resources.Memory), - "NanoCPUs": fmt.Sprintf("%d", c.HostConfig.Resources.NanoCPUs), - "CPUPeriod": fmt.Sprintf("%d", c.HostConfig.Resources.CPUPeriod), - "CPUQuota": fmt.Sprintf("%d", c.HostConfig.Resources.CPUQuota), - "CPURealtimePeriod": fmt.Sprintf("%d", c.HostConfig.Resources.CPURealtimePeriod), - "CPURealtimeRuntime": fmt.Sprintf("%d", c.HostConfig.Resources.CPURealtimeRuntime), - "CpusetCpus": c.HostConfig.Resources.CpusetCpus, - "CpusetMems": c.HostConfig.Resources.CpusetMems, - "DiskQuota": fmt.Sprintf("%d", c.HostConfig.Resources.DiskQuota), - "KernelMemory": fmt.Sprintf("%d", c.HostConfig.Resources.KernelMemory), - "MemoryReservation": fmt.Sprintf("%d", c.HostConfig.Resources.MemoryReservation), - "MemorySwap": fmt.Sprintf("%d", c.HostConfig.Resources.MemorySwap), - "MemorySwappiness": fmt.Sprintf("%d", *c.HostConfig.Resources.MemorySwappiness), - "OomKillDisable": fmt.Sprintf("%t", *c.HostConfig.Resources.OomKillDisable), - "PidsLimit": fmt.Sprintf("%d", c.HostConfig.Resources.PidsLimit), + "hostname": c.Config.Hostname, + "domainname": c.Config.Domainname, + "user": c.Config.User, + "image": c.Config.Image, + "attachstdin": fmt.Sprintf("%t", c.Config.AttachStdin), + "attachstdout": fmt.Sprintf("%t", c.Config.AttachStdout), + "attachstderr": fmt.Sprintf("%t", c.Config.AttachStderr), + "tty": fmt.Sprintf("%t", c.Config.Tty), + "openstdin": fmt.Sprintf("%t", c.Config.OpenStdin), + "stdinonce": fmt.Sprintf("%t", c.Config.StdinOnce), + "privileged": fmt.Sprintf("%t", c.HostConfig.Privileged), + "publishallports": fmt.Sprintf("%t", c.HostConfig.PublishAllPorts), + "readonlyrootfs": fmt.Sprintf("%t", c.HostConfig.ReadonlyRootfs), + "shmsize": fmt.Sprintf("%d", c.HostConfig.ShmSize), + "capadd": strings.Join(c.HostConfig.CapAdd, ", "), + "capdrop": strings.Join(c.HostConfig.CapDrop, ", "), + "runtime": c.HostConfig.Runtime, + "cpushares": fmt.Sprintf("%d", c.HostConfig.Resources.CPUShares), + "memory": fmt.Sprintf("%d", c.HostConfig.Resources.Memory), + "nanocpus": fmt.Sprintf("%d", c.HostConfig.Resources.NanoCPUs), + "cpuperiod": fmt.Sprintf("%d", c.HostConfig.Resources.CPUPeriod), + "cpuquota": fmt.Sprintf("%d", c.HostConfig.Resources.CPUQuota), + "cpurealtimeperiod": fmt.Sprintf("%d", c.HostConfig.Resources.CPURealtimePeriod), + "cpurealtimeruntime": fmt.Sprintf("%d", c.HostConfig.Resources.CPURealtimeRuntime), + "cpusetcpus": c.HostConfig.Resources.CpusetCpus, + "cpusetmems": c.HostConfig.Resources.CpusetMems, + "diskquota": fmt.Sprintf("%d", c.HostConfig.Resources.DiskQuota), + "kernelmemory": fmt.Sprintf("%d", c.HostConfig.Resources.KernelMemory), + "memoryreservation": fmt.Sprintf("%d", c.HostConfig.Resources.MemoryReservation), + "memoryswap": fmt.Sprintf("%d", c.HostConfig.Resources.MemorySwap), + "memoryswappiness": fmt.Sprintf("%d", *c.HostConfig.Resources.MemorySwappiness), + "oomkilldisable": fmt.Sprintf("%t", *c.HostConfig.Resources.OomKillDisable), + "pidslimit": fmt.Sprintf("%d", c.HostConfig.Resources.PidsLimit), } - logging.Debug("[Container Info] Harvested [%+v]", result) + logging.Debug("[Container Info] Harvested [%+v]\n", result) return result, nil } @@ -108,6 +108,6 @@ func ports(m nat.PortSet) []string { } func init() { - logging.Debug("[Container Info] Initialising capability: %s", containerCapability.Title) + logging.Debug("[Container Info] Initialising capability: %s\n", containerCapability.Title) registry.Registry.Add(containerCapability) } From 3c9c91a8e21ea0db13a9bac58cc8c5fdaaa6afb3 Mon Sep 17 00:00:00 2001 From: John McCabe Date: Thu, 21 Sep 2017 14:25:44 +0100 Subject: [PATCH 4/5] (DI-489) Remove unused port fn and filter ints This commit removes an ports function and also stores the ints in the container capability as native JSON values. *NOTE* still storing bools as strings pending an update on the reporter UI side. --- capabilities/container/container.go | 44 ++++++++++------------------ capabilities/payloadfilter/filter.go | 3 +- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/capabilities/container/container.go b/capabilities/container/container.go index 0cef7de..434a701 100644 --- a/capabilities/container/container.go +++ b/capabilities/container/container.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" - "github.com/docker/go-connections/nat" "github.com/puppetlabs/lumogon/capabilities/payloadfilter" "github.com/puppetlabs/lumogon/capabilities/registry" "github.com/puppetlabs/lumogon/dockeradapter" @@ -29,13 +28,13 @@ var containerCapability = dockeradapter.DockerAPICapability{ logging.Debug("[Container Info] Harvesting container information associated with %s [%s]\n", target.Name, target.ID) capability.HarvestID = id - version, err := InspectContainer(client, target.ID) + containerInfo, err := InspectContainer(client, target.ID) if err != nil { capability.PayloadError(err.Error()) return } - filtered, _ := payloadfilter.Filter(version) + filtered, _ := payloadfilter.Filter(containerInfo) capability.Payload = filtered }, @@ -69,26 +68,26 @@ func InspectContainer(client dockeradapter.Harvester, targetID string) (map[stri "privileged": fmt.Sprintf("%t", c.HostConfig.Privileged), "publishallports": fmt.Sprintf("%t", c.HostConfig.PublishAllPorts), "readonlyrootfs": fmt.Sprintf("%t", c.HostConfig.ReadonlyRootfs), - "shmsize": fmt.Sprintf("%d", c.HostConfig.ShmSize), + "shmsize": c.HostConfig.ShmSize, "capadd": strings.Join(c.HostConfig.CapAdd, ", "), "capdrop": strings.Join(c.HostConfig.CapDrop, ", "), "runtime": c.HostConfig.Runtime, - "cpushares": fmt.Sprintf("%d", c.HostConfig.Resources.CPUShares), - "memory": fmt.Sprintf("%d", c.HostConfig.Resources.Memory), - "nanocpus": fmt.Sprintf("%d", c.HostConfig.Resources.NanoCPUs), - "cpuperiod": fmt.Sprintf("%d", c.HostConfig.Resources.CPUPeriod), - "cpuquota": fmt.Sprintf("%d", c.HostConfig.Resources.CPUQuota), - "cpurealtimeperiod": fmt.Sprintf("%d", c.HostConfig.Resources.CPURealtimePeriod), - "cpurealtimeruntime": fmt.Sprintf("%d", c.HostConfig.Resources.CPURealtimeRuntime), + "cpushares": c.HostConfig.Resources.CPUShares, + "memory": c.HostConfig.Resources.Memory, + "nanocpus": c.HostConfig.Resources.NanoCPUs, + "cpuperiod": c.HostConfig.Resources.CPUPeriod, + "cpuquota": c.HostConfig.Resources.CPUQuota, + "cpurealtimeperiod": c.HostConfig.Resources.CPURealtimePeriod, + "cpurealtimeruntime": c.HostConfig.Resources.CPURealtimeRuntime, "cpusetcpus": c.HostConfig.Resources.CpusetCpus, "cpusetmems": c.HostConfig.Resources.CpusetMems, - "diskquota": fmt.Sprintf("%d", c.HostConfig.Resources.DiskQuota), - "kernelmemory": fmt.Sprintf("%d", c.HostConfig.Resources.KernelMemory), - "memoryreservation": fmt.Sprintf("%d", c.HostConfig.Resources.MemoryReservation), - "memoryswap": fmt.Sprintf("%d", c.HostConfig.Resources.MemorySwap), - "memoryswappiness": fmt.Sprintf("%d", *c.HostConfig.Resources.MemorySwappiness), + "diskquota": c.HostConfig.Resources.DiskQuota, + "kernelmemory": c.HostConfig.Resources.KernelMemory, + "memoryreservation": c.HostConfig.Resources.MemoryReservation, + "memoryswap": c.HostConfig.Resources.MemorySwap, + "memoryswappiness": *c.HostConfig.Resources.MemorySwappiness, "oomkilldisable": fmt.Sprintf("%t", *c.HostConfig.Resources.OomKillDisable), - "pidslimit": fmt.Sprintf("%d", c.HostConfig.Resources.PidsLimit), + "pidslimit": c.HostConfig.Resources.PidsLimit, } logging.Debug("[Container Info] Harvested [%+v]\n", result) @@ -96,17 +95,6 @@ func InspectContainer(client dockeradapter.Harvester, targetID string) (map[stri return result, nil } -func ports(m nat.PortSet) []string { - keys := make([]string, len(m)) - - i := 0 - for k := range m { - keys[i] = k.Port() - i++ - } - return keys -} - func init() { logging.Debug("[Container Info] Initialising capability: %s\n", containerCapability.Title) registry.Registry.Add(containerCapability) diff --git a/capabilities/payloadfilter/filter.go b/capabilities/payloadfilter/filter.go index 366a513..0333ce2 100644 --- a/capabilities/payloadfilter/filter.go +++ b/capabilities/payloadfilter/filter.go @@ -36,7 +36,8 @@ func filterMap(value map[string]interface{}) (map[string]interface{}, error) { if coerced != "" { result[key] = coerced } - + case int, int8, int16, int32, int64: + result[key] = coerced case map[string]interface{}: nested, err := filterMap(coerced) if err != nil { From 8f92f04be4a5166432d6b2221c5bdb2c07be97de Mon Sep 17 00:00:00 2001 From: John McCabe Date: Thu, 21 Sep 2017 15:30:00 +0100 Subject: [PATCH 5/5] (DI-489) Update filter tests post int handling Update the filter tests after adding support for native ints in JSON. --- capabilities/payloadfilter/filter_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/capabilities/payloadfilter/filter_test.go b/capabilities/payloadfilter/filter_test.go index 9c319c1..10be0e9 100644 --- a/capabilities/payloadfilter/filter_test.go +++ b/capabilities/payloadfilter/filter_test.go @@ -38,15 +38,19 @@ var filterTests = []filterTest{ false, }, { - "A map with simple non-string values should fail with an error", + "Native JSON ints should be passed through unchanged", map[string]interface{}{ "a": 1, "b": 2, "c": "3", "d": "4", }, - map[string]interface{}{}, - true, + map[string]interface{}{ + "a": 1, + "b": 2, + "c": "3", + "d": "4"}, + false, }, { "A map with complex non-string values should fail with an error",