From 877c68cba0adeb26b7ee37d42403d0cce3cbdecc Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Mon, 4 Mar 2019 08:48:47 -0500 Subject: [PATCH 1/2] Move pkg/util default storage functions from libpod to containers/storage In an effort to remove cross vendoring, trying to fix buildah from importing from libpod. I beleive these libraries make more sense in containers/storage then in libpod. Signed-off-by: Daniel J Walsh --- cmd/containers-storage/main.go | 2 +- store.go | 32 +++-- utils.go | 237 +++++++++++++++++++++++++++++++++ 3 files changed, 260 insertions(+), 11 deletions(-) create mode 100644 utils.go diff --git a/cmd/containers-storage/main.go b/cmd/containers-storage/main.go index f9440b2d8d..4c61db1d09 100644 --- a/cmd/containers-storage/main.go +++ b/cmd/containers-storage/main.go @@ -67,7 +67,7 @@ func main() { } if options.GraphRoot == "" && options.RunRoot == "" && options.GraphDriverName == "" && len(options.GraphDriverOptions) == 0 { - options = storage.DefaultStoreOptions + options, _ = storage.DefaultStoreOptions(false, 0) } args := flags.Args() if len(args) < 1 { diff --git a/store.go b/store.go index 57be441b04..7e39e39598 100644 --- a/store.go +++ b/store.go @@ -32,7 +32,7 @@ import ( var ( // DefaultStoreOptions is a reasonable default set of options. - DefaultStoreOptions StoreOptions + defaultStoreOptions StoreOptions stores []*store storesLock sync.Mutex ) @@ -550,7 +550,7 @@ type store struct { // } func GetStore(options StoreOptions) (Store, error) { if options.RunRoot == "" && options.GraphRoot == "" && options.GraphDriverName == "" && len(options.GraphDriverOptions) == 0 { - options = DefaultStoreOptions + options = defaultStoreOptions } if options.GraphRoot != "" { @@ -3217,8 +3217,20 @@ func copyStringInterfaceMap(m map[string]interface{}) map[string]interface{} { return ret } -// DefaultConfigFile path to the system wide storage.conf file -const DefaultConfigFile = "/etc/containers/storage.conf" +// defaultConfigFile path to the system wide storage.conf file +const defaultConfigFile = "/etc/containers/storage.conf" + +// DefaultConfigFile returns the path to the storage config file used +func DefaultConfigFile(rootless bool) (string, error) { + if rootless { + home, err := homeDir() + if err != nil { + return "", errors.Wrapf(err, "cannot determine users homedir") + } + return filepath.Join(home, ".config/containers/storage.conf"), nil + } + return defaultConfigFile, nil +} // TOML-friendly explicit tables used for conversions. type tomlConfig struct { @@ -3358,19 +3370,19 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) { } func init() { - DefaultStoreOptions.RunRoot = "/var/run/containers/storage" - DefaultStoreOptions.GraphRoot = "/var/lib/containers/storage" - DefaultStoreOptions.GraphDriverName = "" + defaultStoreOptions.RunRoot = "/var/run/containers/storage" + defaultStoreOptions.GraphRoot = "/var/lib/containers/storage" + defaultStoreOptions.GraphDriverName = "" - ReloadConfigurationFile(DefaultConfigFile, &DefaultStoreOptions) + ReloadConfigurationFile(defaultConfigFile, &defaultStoreOptions) } func GetDefaultMountOptions() ([]string, error) { mountOpts := []string{ ".mountopt", - fmt.Sprintf("%s.mountopt", DefaultStoreOptions.GraphDriverName), + fmt.Sprintf("%s.mountopt", defaultStoreOptions.GraphDriverName), } - for _, option := range DefaultStoreOptions.GraphDriverOptions { + for _, option := range defaultStoreOptions.GraphDriverOptions { key, val, err := parsers.ParseKeyValueOpt(option) if err != nil { return nil, err diff --git a/utils.go b/utils.go new file mode 100644 index 0000000000..5dd3d76654 --- /dev/null +++ b/utils.go @@ -0,0 +1,237 @@ +package storage + +import ( + "fmt" + "os" + "os/exec" + "os/user" + "path/filepath" + "strings" + "syscall" + + "github.com/BurntSushi/toml" + "github.com/containers/storage/pkg/idtools" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// ParseIDMapping takes idmappings and subuid and subgid maps and returns a storage mapping +func ParseIDMapping(UIDMapSlice, GIDMapSlice []string, subUIDMap, subGIDMap string) (*IDMappingOptions, error) { + options := IDMappingOptions{ + HostUIDMapping: true, + HostGIDMapping: true, + } + if subGIDMap == "" && subUIDMap != "" { + subGIDMap = subUIDMap + } + if subUIDMap == "" && subGIDMap != "" { + subUIDMap = subGIDMap + } + if len(GIDMapSlice) == 0 && len(UIDMapSlice) != 0 { + GIDMapSlice = UIDMapSlice + } + if len(UIDMapSlice) == 0 && len(GIDMapSlice) != 0 { + UIDMapSlice = GIDMapSlice + } + if len(UIDMapSlice) == 0 && subUIDMap == "" && os.Getuid() != 0 { + UIDMapSlice = []string{fmt.Sprintf("0:%d:1", os.Getuid())} + } + if len(GIDMapSlice) == 0 && subGIDMap == "" && os.Getuid() != 0 { + GIDMapSlice = []string{fmt.Sprintf("0:%d:1", os.Getgid())} + } + + if subUIDMap != "" && subGIDMap != "" { + mappings, err := idtools.NewIDMappings(subUIDMap, subGIDMap) + if err != nil { + return nil, errors.Wrapf(err, "failed to create NewIDMappings for uidmap=%s gidmap=%s", subUIDMap, subGIDMap) + } + options.UIDMap = mappings.UIDs() + options.GIDMap = mappings.GIDs() + } + parsedUIDMap, err := idtools.ParseIDMap(UIDMapSlice, "UID") + if err != nil { + return nil, errors.Wrapf(err, "failed to create ParseUIDMap UID=%s", UIDMapSlice) + } + parsedGIDMap, err := idtools.ParseIDMap(GIDMapSlice, "GID") + if err != nil { + return nil, errors.Wrapf(err, "failed to create ParseGIDMap GID=%s", UIDMapSlice) + } + options.UIDMap = append(options.UIDMap, parsedUIDMap...) + options.GIDMap = append(options.GIDMap, parsedGIDMap...) + if len(options.UIDMap) > 0 { + options.HostUIDMapping = false + } + if len(options.GIDMap) > 0 { + options.HostGIDMapping = false + } + return &options, nil +} + +// GetRootlessRuntimeDir returns the runtime directory when running as non root +func GetRootlessRuntimeDir(rootlessUid int) (string, error) { + runtimeDir := os.Getenv("XDG_RUNTIME_DIR") + if runtimeDir == "" { + tmpDir := fmt.Sprintf("/run/user/%d", rootlessUid) + st, err := os.Stat(tmpDir) + if err == nil && int(st.Sys().(*syscall.Stat_t).Uid) == os.Getuid() && st.Mode().Perm() == 0700 { + return tmpDir, nil + } + } + tmpDir := fmt.Sprintf("%s/%d", os.TempDir(), rootlessUid) + if err := os.MkdirAll(tmpDir, 0700); err != nil { + logrus.Errorf("failed to create %s: %v", tmpDir, err) + } else { + st, err := os.Stat(tmpDir) + if err == nil && int(st.Sys().(*syscall.Stat_t).Uid) == os.Getuid() && st.Mode().Perm() == 0700 { + return tmpDir, nil + } + } + home, err := homeDir() + if err != nil { + return "", errors.Wrapf(err, "neither XDG_RUNTIME_DIR nor HOME was set non-empty") + } + resolvedHome, err := filepath.EvalSymlinks(home) + if err != nil { + return "", errors.Wrapf(err, "cannot resolve %s", home) + } + return filepath.Join(resolvedHome, "rundir"), nil +} + +// getRootlessDirInfo returns the parent path of where the storage for containers and +// volumes will be in rootless mode +func getRootlessDirInfo(rootlessUid int) (string, string, error) { + rootlessRuntime, err := GetRootlessRuntimeDir(rootlessUid) + if err != nil { + return "", "", err + } + + dataDir := os.Getenv("XDG_DATA_HOME") + if dataDir == "" { + home, err := homeDir() + if err != nil { + return "", "", errors.Wrapf(err, "neither XDG_DATA_HOME nor HOME was set non-empty") + } + // runc doesn't like symlinks in the rootfs path, and at least + // on CoreOS /home is a symlink to /var/home, so resolve any symlink. + resolvedHome, err := filepath.EvalSymlinks(home) + if err != nil { + return "", "", errors.Wrapf(err, "cannot resolve %s", home) + } + dataDir = filepath.Join(resolvedHome, ".local", "share") + } + return dataDir, rootlessRuntime, nil +} + +// getRootlessStorageOpts returns the storage opts for containers running as non root +func getRootlessStorageOpts(rootlessUid int) (StoreOptions, error) { + var opts StoreOptions + + dataDir, rootlessRuntime, err := getRootlessDirInfo(rootlessUid) + if err != nil { + return opts, err + } + opts.RunRoot = rootlessRuntime + opts.GraphRoot = filepath.Join(dataDir, "containers", "storage") + if path, err := exec.LookPath("fuse-overlayfs"); err == nil { + opts.GraphDriverName = "overlay" + opts.GraphDriverOptions = []string{fmt.Sprintf("overlay.mount_program=%s", path)} + } else { + opts.GraphDriverName = "vfs" + } + return opts, nil +} + +type tomlOptionsConfig struct { + MountProgram string `toml:"mount_program"` +} + +func getTomlStorage(storeOptions *StoreOptions) *tomlConfig { + config := new(tomlConfig) + + config.Storage.Driver = storeOptions.GraphDriverName + config.Storage.RunRoot = storeOptions.RunRoot + config.Storage.GraphRoot = storeOptions.GraphRoot + for _, i := range storeOptions.GraphDriverOptions { + s := strings.Split(i, "=") + if s[0] == "overlay.mount_program" { + config.Storage.Options.MountProgram = s[1] + } + } + + return config +} + +// DefaultStoreOptions returns the default storage ops for containers +func DefaultStoreOptions(rootless bool, rootlessUid int) (StoreOptions, error) { + var ( + defaultRootlessRunRoot string + defaultRootlessGraphRoot string + err error + ) + storageOpts := defaultStoreOptions + if rootless { + storageOpts, err = getRootlessStorageOpts(rootlessUid) + if err != nil { + return storageOpts, err + } + } + + storageConf, err := DefaultConfigFile(rootless) + if err != nil { + return storageOpts, err + } + if _, err = os.Stat(storageConf); err == nil { + defaultRootlessRunRoot = storageOpts.RunRoot + defaultRootlessGraphRoot = storageOpts.GraphRoot + storageOpts = StoreOptions{} + ReloadConfigurationFile(storageConf, &storageOpts) + } + + if !os.IsNotExist(err) { + return storageOpts, errors.Wrapf(err, "cannot stat %s", storageConf) + } + + if rootless { + if err == nil { + // If the file did not specify a graphroot or runroot, + // set sane defaults so we don't try and use root-owned + // directories + if storageOpts.RunRoot == "" { + storageOpts.RunRoot = defaultRootlessRunRoot + } + if storageOpts.GraphRoot == "" { + storageOpts.GraphRoot = defaultRootlessGraphRoot + } + } else { + if err := os.MkdirAll(filepath.Dir(storageConf), 0755); err != nil { + return storageOpts, errors.Wrapf(err, "cannot make directory %s", filepath.Dir(storageConf)) + } + file, err := os.OpenFile(storageConf, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + return storageOpts, errors.Wrapf(err, "cannot open %s", storageConf) + } + + tomlConfiguration := getTomlStorage(&storageOpts) + defer file.Close() + enc := toml.NewEncoder(file) + if err := enc.Encode(tomlConfiguration); err != nil { + os.Remove(storageConf) + + return storageOpts, errors.Wrapf(err, "failed to encode %s", storageConf) + } + } + } + return storageOpts, nil +} + +func homeDir() (string, error) { + home := os.Getenv("HOME") + if home == "" { + usr, err := user.Current() + if err != nil { + return "", errors.Wrapf(err, "neither XDG_RUNTIME_DIR nor HOME was set non-empty") + } + home = usr.HomeDir + } + return home, nil +} From 058f984a588046cdb3b30ac148dcf4762b1033bc Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Tue, 19 Mar 2019 09:22:01 -0400 Subject: [PATCH 2/2] Bump to latest golang for testing. Signed-off-by: Daniel J Walsh --- .travis.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index e0bcd4291b..dc1c613912 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,10 +18,7 @@ env: - GO_VERSION="1.11" DISTRO="ubuntu" - - GO_VERSION="1.10" - DISTRO="ubuntu" - - - GO_VERSION="1.9" + - GO_VERSION="1.12" DISTRO="ubuntu" # Fedora @@ -31,10 +28,7 @@ env: - GO_VERSION="1.11" DISTRO="fedora" - - GO_VERSION="1.10" - DISTRO="fedora" - - - GO_VERSION="1.9" + - GO_VERSION="1.12" DISTRO="fedora" # CentOS @@ -44,10 +38,7 @@ env: - GO_VERSION="1.11" DISTRO="centos" - - GO_VERSION="1.10" - DISTRO="centos" - - - GO_VERSION="1.9" + - GO_VERSION="1.12" DISTRO="centos" # GO_VERSION="stable" builds successfully, but tests fail on all platforms.