runc exec: implement CPU affinity

As per
 - opencontainers/runtime-spec#1253
 - opencontainers/runtime-spec#1261

Add some tests (alas it's impossible to test initial CPU affinity
without adding debug logging).

Signed-off-by: Kir Kolyshkin <[email protected]>

libcontainer/configs/config.go

@@ -225,6 +225,9 @@ type Config struct {
 
 	// IOPriority is the container's I/O priority.
 	IOPriority *IOPriority `json:"io_priority,omitempty"`
+
+	// ExecCPUAffinity is CPU affinity for a non-init process to be run in the container.
+	ExecCPUAffinity *CPUAffinity `json:"exec_cpu_affinity,omitempty"`
 }
 
 // Scheduler is based on the Linux sched_setattr(2) syscall.

libcontainer/process_linux.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -122,13 +123,96 @@ func (p *setnsProcess) signal(sig os.Signal) error {
 	return unix.Kill(p.pid(), s)
 }
 
+func affToUnix(str string) (*unix.CPUSet, error) {
+	s := new(unix.CPUSet)
+	for _, r := range strings.Split(str, ",") {
+		// Allow extra spaces around.
+		r = strings.TrimSpace(r)
+		// Allow empty elements (extra commas).
+		if r == "" {
+			continue
+		}
+		if r0, r1, found := strings.Cut(r, "-"); found {
+			start, err := strconv.ParseUint(r0, 10, 32)
+			if err != nil {
+				return nil, err
+			}
+			end, err := strconv.ParseUint(r1, 10, 32)
+			if err != nil {
+				return nil, err
+			}
+			if start > end {
+				return nil, errors.New("invalid range: " + r)
+			}
+			for i := int(start); i <= int(end); i++ {
+				s.Set(i)
+			}
+		} else {
+			val, err := strconv.ParseUint(r, 10, 32)
+			if err != nil {
+				return nil, err
+			}
+			s.Set(int(val))
+		}
+	}
+
+	return s, nil
+}
+
+// Starts setns process with specified initial CPU affinity.
+func (p *setnsProcess) startWithCPUAffinity() error {
+	aff := p.config.Config.ExecCPUAffinity
+	if aff == nil || aff.Initial == "" {
+		return p.cmd.Start()
+	}
+	cpus, err := affToUnix(aff.Initial)
+	if err != nil {
+		return fmt.Errorf("invalid execCPUAffinity.initial: %w", err)
+	}
+
+	errCh := make(chan error)
+	defer close(errCh)
+
+	// Use a goroutine to dedicate an OS thread.
+	go func() {
+		// Don't call runtime.UnlockOSThread to terminate the OS thread
+		// when goroutine exits.
+		runtime.LockOSThread()
+
+		// Command inherits the CPU affinity.
+		if err := unix.SchedSetaffinity(unix.Gettid(), cpus); err != nil {
+			errCh <- fmt.Errorf("setting initial CPU affinity: %w", err)
+			return
+		}
+
+		errCh <- p.cmd.Start()
+	}()
+
+	return <-errCh
+}
+
+func (p *setnsProcess) setFinalCPUAffinity() error {
+	aff := p.config.Config.ExecCPUAffinity
+	if aff == nil || aff.Final == "" {
+		return nil
+	}
+	cpus, err := affToUnix(aff.Final)
+	if err != nil {
+		return fmt.Errorf("invalid execCPUAffinity.final: %w", err)
+	}
+	if err := unix.SchedSetaffinity(p.pid(), cpus); err != nil {
+		return fmt.Errorf("setting final CPU affinity: %w", err)
+	}
+	return nil
+}
+
 func (p *setnsProcess) start() (retErr error) {
 	defer p.comm.closeParent()
 
-	// get the "before" value of oom kill count
+	// Get the "before" value of oom kill count.
 	oom, _ := p.manager.OOMKillCount()
-	err := p.cmd.Start() // https://github.com/opencontainers/runc/pull/3923/commits/afc23e33971b657c4a09c54b16c6139651171aad
-	// close the child-side of the pipes (controlled by child)
+	err := p.startWithCPUAffinity()
+	// Close the child-side of the pipes (controlled by child).
 	p.comm.closeChild()
 	if err != nil {
 		return fmt.Errorf("error starting setns process: %w", err)
@@ -196,6 +280,9 @@ func (p *setnsProcess) start() (retErr error) {
 			}
 		}
 	}
+	if err := p.setFinalCPUAffinity(); err != nil {
+		return err
+	}
 
 	if err := utils.WriteJSON(p.comm.initSockParent, p.config); err != nil {
 		return fmt.Errorf("error writing config to pipe: %w", err)

libcontainer/specconv/spec_linux.go

@@ -539,6 +539,11 @@ func CreateLibcontainerConfig(opts *CreateOpts) (*configs.Config, error) {
 			ioPriority := *spec.Process.IOPriority
 			config.IOPriority = &ioPriority
 		}
+		if spec.Process.ExecCPUAffinity != nil {
+			a := *spec.Process.ExecCPUAffinity
+			config.ExecCPUAffinity = &a
+		}
+
 	}
 	createHooks(spec, config)
 	config.Version = specs.Version

tests/integration/cpu_affinity.bats

@@ -0,0 +1,77 @@
+#!/usr/bin/env bats
+# Exec CPU affinity tests. For more details, see:
+#  - https://github.com/opencontainers/runtime-spec/pull/1253
+
+load helpers
+
+function setup() {
+	requires smp cgroups_cpuset
+	setup_busybox
+}
+
+function teardown() {
+	teardown_bundle
+}
+
+function all_cpus() {
+	cat /sys/devices/system/cpu/online
+}
+
+function first_cpu() {
+	all_cpus | sed 's/[-,].*//g'
+}
+
+@test "runc exec [CPU affinity inherited from runc]" {
+	requires root # For taskset.
+
+	first="$(first_cpu)"
+
+	# Container's process CPU affinity is inherited from that of runc.
+	taskset -p -c "$first" $$
+
+	runc run -d --console-socket "$CONSOLE_SOCKET" ct1
+	[ "$status" -eq 0 ]
+
+	# Check init.
+	runc exec ct1 grep "Cpus_allowed_list:" /proc/1/status
+	[ "$status" -eq 0 ]
+	[[ "${lines[0]}" == "Cpus_allowed_list:	$first" ]]
+
+	# Check exec.
+	runc exec ct1 grep "Cpus_allowed_list:" /proc/self/status
+	[ "$status" -eq 0 ]
+	[[ "${lines[0]}" == "Cpus_allowed_list:	$first" ]]
+}
+
+@test "runc exec [CPU affinity, only initial is set]" {
+	requires root # For taskset.
+
+	first="$(first_cpu)"
+
+	update_config ".process.execCPUAffinity.initial = \"$first\""
+
+	runc run -d --console-socket "$CONSOLE_SOCKET" ct1
+	[ "$status" -eq 0 ]
+
+	runc exec ct1 grep "Cpus_allowed_list:" /proc/self/status
+	[ "$status" -eq 0 ]
+	[[ "${lines[0]}" == "Cpus_allowed_list:	$first" ]]
+}
+
+@test "runc exec [CPU affinity, initial and final are set]" {
+	requires root # For taskset.
+
+	first="$(first_cpu)"
+	second=$((first+1)) # Hacky; might not work in all environments.
+
+	update_config "	  .process.execCPUAffinity.initial = \"$first\"
+			| .process.execCPUAffinity.final = \"$second\""
+
+	taskset -p -c "$first" $$
+	runc run -d --console-socket "$CONSOLE_SOCKET" ct1
+	[ "$status" -eq 0 ]
+
+	runc exec ct1 grep "Cpus_allowed_list:" /proc/self/status
+	[ "$status" -eq 0 ]
+	[[ "${lines[0]}" == "Cpus_allowed_list:	$second" ]]
+}