Merge pull request moby#15862 from calavera/share_shm_and_mqueue

Share shm and mqueue between containers.

Author: Jess Frazelle

daemon/container.go

@@ -288,10 +288,17 @@ func (container *Container) Start() (err error) {
 		return err
 	}
 
+	if !container.hostConfig.IpcMode.IsContainer() && !container.hostConfig.IpcMode.IsHost() {
+		if err := container.setupIpcDirs(); err != nil {
+			return err
+		}
+	}
+
 	mounts, err := container.setupMounts()
 	if err != nil {
 		return err
 	}
+	mounts = append(mounts, container.ipcMounts()...)
 
 	container.command.Mounts = mounts
 	return container.waitForStart()
@@ -330,8 +337,12 @@ func (container *Container) isNetworkAllocated() bool {
 func (container *Container) cleanup() {
 	container.releaseNetwork()
 
+	if err := container.unmountIpcMounts(); err != nil {
+		logrus.Errorf("%s: Failed to umount ipc filesystems: %v", container.ID, err)
+	}
+
 	if err := container.Unmount(); err != nil {
-		logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err)
+		logrus.Errorf("%s: Failed to umount filesystem: %v", container.ID, err)
 	}
 
 	for _, eConfig := range container.execCommands.s {

daemon/container_unix.go

@@ -52,6 +52,8 @@ type Container struct {
 	AppArmorProfile string
 	HostnamePath    string
 	HostsPath       string
+	ShmPath         string
+	MqueuePath      string
 	MountPoints     map[string]*mountPoint
 	ResolvConfPath  string
 
@@ -192,15 +194,31 @@ func populateCommand(c *Container, env []string) error {
 	}
 
 	ipc := &execdriver.Ipc{}
+	var err error
+	c.ShmPath, err = c.shmPath()
+	if err != nil {
+		return err
+	}
+
+	c.MqueuePath, err = c.mqueuePath()
+	if err != nil {
+		return err
+	}
 
 	if c.hostConfig.IpcMode.IsContainer() {
 		ic, err := c.getIpcContainer()
 		if err != nil {
 			return err
 		}
 		ipc.ContainerID = ic.ID
+		c.ShmPath = ic.ShmPath
+		c.MqueuePath = ic.MqueuePath
 	} else {
 		ipc.HostIpc = c.hostConfig.IpcMode.IsHost()
+		if ipc.HostIpc {
+			c.ShmPath = "/dev/shm"
+			c.MqueuePath = "/dev/mqueue"
+		}
 	}
 
 	pid := &execdriver.Pid{}
@@ -1236,3 +1254,99 @@ func (container *Container) removeMountPoints(rm bool) error {
 	}
 	return nil
 }
+
+func (container *Container) shmPath() (string, error) {
+	return container.getRootResourcePath("shm")
+}
+func (container *Container) mqueuePath() (string, error) {
+	return container.getRootResourcePath("mqueue")
+}
+
+func (container *Container) setupIpcDirs() error {
+	shmPath, err := container.shmPath()
+	if err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(shmPath, 0700); err != nil {
+		return err
+	}
+
+	if err := syscall.Mount("shm", shmPath, "tmpfs", uintptr(syscall.MS_NOEXEC|syscall.MS_NOSUID|syscall.MS_NODEV), label.FormatMountLabel("mode=1777,size=65536k", container.getMountLabel())); err != nil {
+		return fmt.Errorf("mounting shm tmpfs: %s", err)
+	}
+
+	mqueuePath, err := container.mqueuePath()
+	if err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(mqueuePath, 0700); err != nil {
+		return err
+	}
+
+	if err := syscall.Mount("mqueue", mqueuePath, "mqueue", uintptr(syscall.MS_NOEXEC|syscall.MS_NOSUID|syscall.MS_NODEV), ""); err != nil {
+		return fmt.Errorf("mounting mqueue mqueue : %s", err)
+	}
+
+	return nil
+}
+
+func (container *Container) unmountIpcMounts() error {
+	if container.hostConfig.IpcMode.IsContainer() || container.hostConfig.IpcMode.IsHost() {
+		return nil
+	}
+
+	var errors []string
+	shmPath, err := container.shmPath()
+	if err != nil {
+		logrus.Error(err)
+		errors = append(errors, err.Error())
+	} else {
+		if err := detachMounted(shmPath); err != nil {
+			logrus.Errorf("failed to umount %s: %v", shmPath, err)
+			errors = append(errors, err.Error())
+		}
+
+	}
+
+	mqueuePath, err := container.mqueuePath()
+	if err != nil {
+		logrus.Error(err)
+		errors = append(errors, err.Error())
+	} else {
+		if err := detachMounted(mqueuePath); err != nil {
+			logrus.Errorf("failed to umount %s: %v", mqueuePath, err)
+			errors = append(errors, err.Error())
+		}
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf("failed to cleanup ipc mounts:\n%v", strings.Join(errors, "\n"))
+	}
+
+	return nil
+}
+
+func (container *Container) ipcMounts() []execdriver.Mount {
+	var mounts []execdriver.Mount
+	label.SetFileLabel(container.ShmPath, container.MountLabel)
+	mounts = append(mounts, execdriver.Mount{
+		Source:      container.ShmPath,
+		Destination: "/dev/shm",
+		Writable:    true,
+		Private:     true,
+	})
+	label.SetFileLabel(container.MqueuePath, container.MountLabel)
+	mounts = append(mounts, execdriver.Mount{
+		Source:      container.MqueuePath,
+		Destination: "/dev/mqueue",
+		Writable:    true,
+		Private:     true,
+	})
+	return mounts
+}
+
+func detachMounted(path string) error {
+	return syscall.Unmount(path, syscall.MNT_DETACH)
+}

daemon/container_windows.go

@@ -170,3 +170,15 @@ func (container *Container) prepareMountPoints() error {
 func (container *Container) removeMountPoints(_ bool) error {
 	return nil
 }
+
+func (container *Container) setupIpcDirs() error {
+	return nil
+}
+
+func (container *Container) unmountIpcMounts() error {
+	return nil
+}
+
+func (container *Container) ipcMounts() []execdriver.Mount {
+	return nil
+}

daemon/daemon.go

@@ -217,6 +217,9 @@ func (daemon *Daemon) Register(container *Container) error {
 		}
 		daemon.execDriver.Terminate(cmd)
 
+		if err := container.unmountIpcMounts(); err != nil {
+			logrus.Errorf("%s: Failed to umount ipc filesystems: %v", container.ID, err)
+		}
 		if err := container.Unmount(); err != nil {
 			logrus.Debugf("unmount error %s", err)
 		}
@@ -766,6 +769,11 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo
 	d.EventsService = eventsService
 	d.volumes = volStore
 	d.root = config.Root
+
+	if err := d.cleanupMounts(); err != nil {
+		return nil, err
+	}
+
 	go d.execCommandGC()
 
 	if err := d.restore(); err != nil {
@@ -848,6 +856,10 @@ func (daemon *Daemon) Shutdown() error {
 		}
 	}
 
+	if err := daemon.cleanupMounts(); err != nil {
+		return err
+	}
+
 	return nil
 }
 

daemon/daemon_linux.go

@@ -0,0 +1,56 @@
+package daemon
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/Sirupsen/logrus"
+)
+
+// cleanupMounts umounts shm/mqueue mounts for old containers
+func (daemon *Daemon) cleanupMounts() error {
+	logrus.Debugf("Cleaning up old shm/mqueue mounts: start.")
+	f, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return daemon.cleanupMountsFromReader(f, detachMounted)
+}
+
+func (daemon *Daemon) cleanupMountsFromReader(reader io.Reader, unmount func(target string) error) error {
+	sc := bufio.NewScanner(reader)
+	var errors []string
+	for sc.Scan() {
+		line := sc.Text()
+		fields := strings.Fields(line)
+		if strings.HasPrefix(fields[4], daemon.repository) {
+			logrus.Debugf("Mount base: %v, repository %s", fields[4], daemon.repository)
+			mnt := fields[4]
+			mountBase := filepath.Base(mnt)
+			if mountBase == "mqueue" || mountBase == "shm" {
+				logrus.Debugf("Unmounting %v", mnt)
+				if err := unmount(mnt); err != nil {
+					logrus.Error(err)
+					errors = append(errors, err.Error())
+				}
+			}
+		}
+	}
+
+	if err := sc.Err(); err != nil {
+		return err
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf("Error cleaningup mounts:\n%v", strings.Join(errors, "\n"))
+	}
+
+	logrus.Debugf("Cleaning up old shm/mqueue mounts: done.")
+	return nil
+}

daemon/daemon_linux_test.go

@@ -0,0 +1,58 @@
+// +build linux
+
+package daemon
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestCleanupMounts(t *testing.T) {
+	fixture := `230 138 0:60 / / rw,relatime - overlay overlay rw,lowerdir=/var/lib/docker/overlay/0ef9f93d5d365c1385b09d54bbee6afff3d92002c16f22eccb6e1549b2ff97d8/root,upperdir=/var/lib/docker/overlay/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb/upper,workdir=/var/lib/docker/overlay/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb/work
+231 230 0:56 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
+232 230 0:57 / /dev rw,nosuid - tmpfs tmpfs rw,mode=755
+233 232 0:58 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+234 232 0:59 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k
+235 232 0:55 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw
+236 230 0:61 / /sys rw,nosuid,nodev,noexec,relatime - sysfs sysfs rw
+237 236 0:62 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw
+238 237 0:21 /system.slice/docker.service /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd
+239 237 0:23 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/perf_event rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,perf_event
+240 237 0:24 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpuset,clone_children
+241 237 0:25 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,devices
+242 237 0:26 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/freezer rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,freezer
+243 237 0:27 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpu,cpuacct
+244 237 0:28 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/blkio rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,blkio
+245 237 0:29 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/net_cls,net_prio rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,net_cls,net_prio
+246 237 0:30 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/hugetlb rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,hugetlb
+247 237 0:31 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,memory
+248 230 253:1 /var/lib/docker/volumes/510cc41ac68c48bd4eac932e3e09711673876287abf1b185312cfbfe6261a111/_data /var/lib/docker rw,relatime - ext4 /dev/disk/by-uuid/ba70ea0c-1a8f-4ee4-9687-cb393730e2b5 rw,errors=remount-ro,data=ordered
+250 230 253:1 /var/lib/docker/containers/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb/hostname /etc/hostname rw,relatime - ext4 /dev/disk/by-uuid/ba70ea0c-1a8f-4ee4-9687-cb393730e2b5 rw,errors=remount-ro,data=ordered
+251 230 253:1 /var/lib/docker/containers/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb/hosts /etc/hosts rw,relatime - ext4 /dev/disk/by-uuid/ba70ea0c-1a8f-4ee4-9687-cb393730e2b5 rw,errors=remount-ro,data=ordered
+252 232 0:13 /1 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=000
+139 236 0:11 / /sys/kernel/security rw,relatime - securityfs none rw
+140 230 0:54 / /tmp rw,relatime - tmpfs none rw
+145 230 0:3 / /run/docker/netns/default rw - nsfs nsfs rw
+130 140 0:45 / /tmp/docker_recursive_mount_test312125472/tmpfs rw,relatime - tmpfs tmpfs rw
+131 230 0:3 / /run/docker/netns/47903e2e6701 rw - nsfs nsfs rw
+133 230 0:55 / /go/src/github.com/docker/docker/bundles/1.9.0-dev/test-integration-cli/d45526097/graph/containers/47903e2e67014246eba27607809d5f5c2437c3bf84c2986393448f84093cc40b/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw`
+
+	d := &Daemon{
+		repository: "/go/src/github.com/docker/docker/bundles/1.9.0-dev/test-integration-cli/d45526097/graph/containers/",
+	}
+
+	expected := "/go/src/github.com/docker/docker/bundles/1.9.0-dev/test-integration-cli/d45526097/graph/containers/47903e2e67014246eba27607809d5f5c2437c3bf84c2986393448f84093cc40b/mqueue"
+	var unmounted bool
+	unmount := func(target string) error {
+		if target == expected {
+			unmounted = true
+		}
+		return nil
+	}
+
+	d.cleanupMountsFromReader(strings.NewReader(fixture), unmount)
+
+	if !unmounted {
+		t.Fatalf("Expected to unmount the mqueue")
+	}
+}

daemon/daemon_windows.go

@@ -146,3 +146,7 @@ func (daemon *Daemon) newBaseContainer(id string) Container {
 		},
 	}
 }
+
+func (daemon *Daemon) cleanupMounts() error {
+	return nil
+}

daemon/execdriver/native/template/default_template.go

@@ -61,19 +61,6 @@ func New() *configs.Config {
 				Flags:       syscall.MS_NOSUID | syscall.MS_NOEXEC,
 				Data:        "newinstance,ptmxmode=0666,mode=0620,gid=5",
 			},
-			{
-				Device:      "tmpfs",
-				Source:      "shm",
-				Destination: "/dev/shm",
-				Data:        "mode=1777,size=65536k",
-				Flags:       defaultMountFlags,
-			},
-			{
-				Source:      "mqueue",
-				Destination: "/dev/mqueue",
-				Device:      "mqueue",
-				Flags:       defaultMountFlags,
-			},
 			{
 				Source:      "sysfs",
 				Destination: "/sys",

integration-cli/docker_cli_daemon_test.go

@@ -1475,9 +1475,11 @@ func (s *DockerDaemonSuite) TestCleanupMountsAfterCrash(c *check.C) {
 	id := strings.TrimSpace(out)
 	c.Assert(s.d.cmd.Process.Signal(os.Kill), check.IsNil)
 	c.Assert(s.d.Start(), check.IsNil)
-	mountOut, err := exec.Command("mount").CombinedOutput()
+	mountOut, err := ioutil.ReadFile("/proc/self/mountinfo")
 	c.Assert(err, check.IsNil, check.Commentf("Output: %s", mountOut))
-	c.Assert(strings.Contains(string(mountOut), id), check.Equals, false, check.Commentf("Something mounted from older daemon start: %s", mountOut))
+
+	comment := check.Commentf("%s is still mounted from older daemon start:\nDaemon root repository %s\n%s", id, s.d.folder, mountOut)
+	c.Assert(strings.Contains(string(mountOut), id), check.Equals, false, comment)
 }
 
 func (s *DockerDaemonSuite) TestRunContainerWithBridgeNone(c *check.C) {