diff --git a/CHANGELOG.md b/CHANGELOG.md index cb2c3f0c..8eb3d610 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,25 @@ The format is based on [Keep a Changelog][], and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). [Keep a Changelog]: https://keepachangelog.com/en/1.0.0/ -[Unreleased]: https://github.com/yourbase/yb/compare/v0.5.6...HEAD +[Unreleased]: https://github.com/yourbase/yb/compare/v0.5.7...HEAD + +## [0.5.7][] - 2021-03-01 + +Version 0.5.7 backports a fix for a locale environment variable issue. + +[0.5.7]: https://github.com/yourbase/yb/releases/tag/v0.5.7 + +### Changed + +- The build environment now sets `LANG` and other locale environment variables + to `C.UTF-8` or the closest approximation thereof. Previously, these + variables were unset, which caused problems with programs that required a + UTF-8 character set to function properly, like those written in Ruby or Python. + +### Fixed + +- The `TZ` environment variable is now set to `UTC0` by default. Previously, + it was set to `UTC`, which is not a POSIX-conforming value. ## [0.5.6][] - 2021-02-11 diff --git a/internal/biome/README.md b/internal/biome/README.md new file mode 100644 index 00000000..15351a31 --- /dev/null +++ b/internal/biome/README.md @@ -0,0 +1,66 @@ +# Build Environment Reference + +**WIP specification. This information should eventually live in end-user +reference documentation.** + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", +"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be +interpreted as described in [RFC 2119][]. + +[RFC 2119]: https://tools.ietf.org/html/rfc2119 + +## Environment Variables + +Each target **SHALL** run in a separate, isolated environment. For example, +a target may run in a Docker container or a chroot jail. A target _MAY_ run on +a different host from the build runner. At a minimum, the following environment +variables **MUST** be set for commands running in POSIX environments: + +- `HOME` **MUST** be set to the path of an readable and writable directory. + This _SHOULD NOT_ be the same as the user's actual `HOME` directory to + keep builds reproducible. +- `LOGNAME` and `USER` **MUST** be set to the name of the POSIX user running + the command (not the runner of `yb`). +- `PATH` **MUST NOT** be empty. +- `TZ` **MUST** be set to `UTC0`. +- `LANG` _SHOULD_ be set to `C.UTF-8` if the environment supports it. + Otherwise, `LANG` **MUST** be set to `C`. +- One of `LC_ALL` or `LC_CTYPE` **MUST** be set to C-like locale category + whose `charmap` is UTF-8. If `LC_CTYPE` is set, `LC_ALL` **MUST NOT** be set. + +### Examples of Locale Settings + +For Linux systems: + +``` +LANG=C.UTF-8 +LC_ALL=C.UTF-8 +``` + +For macOS systems: + +``` +LANG=C +LC_CTYPE=UTF-8 +``` + +### Further Reading + +- [POSIX.1-2017 Environment Variables](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html) + describes the meaning of the standard environment variables. +- [PEP 538](https://www.python.org/dev/peps/pep-0538/) documents Python's + process for bootstrapping a UTF-8 locale with rationale and platform-specific + caveats. + +## Expected Userspace + +The build environment that a target's commands run in **MUST** include the +standard [POSIX utilities][]. yb also depends on the following utilities being +available: + +- `python` on non-Linux to fill in `readlink --canonicalize-existing` behavior +- `readlink` on Linux +- `tar` +- `unzip` + +[POSIX utilities]: https://pubs.opengroup.org/onlinepubs/9699919799/idx/utilities.html diff --git a/internal/biome/biome.go b/internal/biome/biome.go index 08f21db0..2c918f0d 100644 --- a/internal/biome/biome.go +++ b/internal/biome/biome.go @@ -195,8 +195,8 @@ func (l Local) Run(ctx context.Context, invoke *Invocation) error { "HOME=" + l.HomeDir, "LOGNAME=" + os.Getenv("LOGNAME"), "USER=" + os.Getenv("USER"), - "TZ=UTC", } + c.Env = appendStandardEnv(c.Env, runtime.GOOS) c.Env = invoke.Env.appendTo(c.Env, os.Getenv("PATH"), filepath.ListSeparator) c.Dir = dir c.Stdin = invoke.Stdin @@ -208,6 +208,16 @@ func (l Local) Run(ctx context.Context, invoke *Invocation) error { return nil } +func appendStandardEnv(env []string, biomeOS string) []string { + env = append(env, "TZ=UTC0") + if biomeOS == MacOS { + env = append(env, "LANG=C", "LC_CTYPE=UTF-8") + } else { + env = append(env, "LANG=C.UTF-8", "LC_ALL=C.UTF-8") + } + return env +} + func (l Local) lookPath(env Environment, dir string, program string) (string, error) { abs := func(path string) string { if filepath.IsAbs(path) { diff --git a/internal/biome/biome_test.go b/internal/biome/biome_test.go index 60846fe5..199d656f 100644 --- a/internal/biome/biome_test.go +++ b/internal/biome/biome_test.go @@ -22,6 +22,8 @@ import ( "os" "os/exec" "path/filepath" + "runtime" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -157,6 +159,84 @@ func TestLocal(t *testing.T) { } } +func TestStandardEnv(t *testing.T) { + stdenv := appendStandardEnv(nil, runtime.GOOS) + + t.Run("TZ", func(t *testing.T) { + found := false + for _, e := range stdenv { + const prefix = "TZ=" + if !strings.HasPrefix(e, prefix) { + continue + } + found = true + if got, want := e[len(prefix):], "UTC0"; got != want { + t.Errorf("TZ = %q; want %q", got, want) + } + } + if !found { + t.Error("TZ not set") + } + }) + + t.Run("LANG", func(t *testing.T) { + found := false + for _, e := range stdenv { + const prefix = "LANG=" + if !strings.HasPrefix(e, prefix) { + continue + } + found = true + if got, want1, want2 := e[len(prefix):], "C.UTF-8", "C"; got != want1 && got != want2 { + t.Errorf("LANG = %q; want %q or %q", got, want1, want2) + } + } + if !found { + t.Error("LANG not set") + } + }) + + t.Run("Charmap", func(t *testing.T) { + // Run locale tool to get character encoding. + // https://pubs.opengroup.org/onlinepubs/9699919799/utilities/locale.html + c := exec.Command("locale", "-k", "charmap") + c.Env = stdenv + stdout := new(strings.Builder) + stderr := new(strings.Builder) + c.Stdout = stdout + c.Stderr = stderr + err := c.Run() + if stderr.Len() > 0 { + t.Logf("stderr:\n%s", stderr) + } + if err != nil { + t.Error("locale:", err) + } + got := parseLocaleOutput(stdout.String())["charmap"] + const want = "UTF-8" + if got != want { + t.Errorf("charmap = %q; want %q", got, want) + } + }) +} + +func parseLocaleOutput(out string) map[string]string { + m := make(map[string]string) + for _, line := range strings.Split(out, "\n") { + eq := strings.Index(line, "=") + if eq == -1 { + continue + } + k, v := line[:eq], line[eq+1:] + if strings.HasPrefix(v, `"`) { + v = strings.TrimPrefix(v, `"`) + v = strings.TrimSuffix(v, `"`) + } + m[k] = v + } + return m +} + func TestExecPrefix(t *testing.T) { tests := []struct { name string diff --git a/internal/biome/docker.go b/internal/biome/docker.go index f626dfad..2ce1e932 100644 --- a/internal/biome/docker.go +++ b/internal/biome/docker.go @@ -240,8 +240,8 @@ func (c *Container) Run(ctx context.Context, invoke *Invocation) error { opts.Env = []string{ // TODO(light): Set LOGNAME and USER. "HOME=" + c.dirs.Home, - "TZ=UTC", } + opts.Env = appendStandardEnv(opts.Env, c.Describe().OS) opts.Env = invoke.Env.appendTo(opts.Env, c.path, ':') if slashpath.IsAbs(invoke.Dir) { opts.WorkingDir = invoke.Dir