Skip to content

Commit

Permalink
support reading stdout/stderr from streams
Browse files Browse the repository at this point in the history
It can be useful to let the user run the test command(s) itself, for example
when complex shell scripts are involved which need to invoke `go test` multiple
times.

With -stdin, the `go test` stdout is expected on stdin of gotestsum, so it can
be used in a pipe. To detect abnormal termination of the test commands, bash
with "set -o pipefail" should be used. Beware that such failures are not
detected by gotestsum.

To also capture stderr with gotestsum, stderr must get redirected like this:

   mkfifo /tmp/pipe
   go test ... 2>/tmp/pipe | gotestsum -stdin -stderr 3 3</tmp/pipe
  • Loading branch information
pohly committed Jun 16, 2024
1 parent 8253d0e commit 8475e03
Show file tree
Hide file tree
Showing 8 changed files with 220 additions and 7 deletions.
36 changes: 36 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ source with `go install gotest.tools/gotestsum@latest`. To run without installin
- Print a [summary](#summary) of the test run after running all the tests.
- Use any [`go test` flag](#custom-go-test-command),
run a script with [`--raw-command`](#custom-go-test-command),
run `go test` manually and [pipe the output](#pipe) into `gotestsum`
or [run a compiled test binary](#executing-a-compiled-test-binary).

**CI and Automation**
Expand Down Expand Up @@ -306,6 +307,41 @@ gotestsum --raw-command ./profile.sh ./...
TEST_DIRECTORY=./io/http gotestsum
```

### Pipe into gotestsum

When using a shell script which decides how to invoke `go test`, it can be
difficult to generate a script for use with `--raw-command`. A more natural
approach in a shell script is using a pipe:

**Example: simple pipe**
```
go test . | gotestsum --stdin
```

As with `--raw-command` above, only `test2json` output is allowed on
stdin. Anything else causes `gotestsum` to fail with a parser error.

In this simple example, stderr of the test goes to the console and is not
captured by `gotestsum`. To get that behavior, stderr of the first command can
be redirected to a named pipe and then be read from there by `gotestsum`:

**Example: redirect stdout and stderr**
```
mkfifo /tmp/stderr-pipe
go test 2>/tmp/stderr-pipe | gotestsum --stdin --stderr 3 3</tmp/stderr-pipe
```

Note that `gotestsum` is not aware of a non-zero exit code of the test
command. Bash's `pipefile` can be used to detect such a failure:

**Example: pipefail**
```
set -o pipefail # bashism
go test . | gotestsum --stdin
res=$? # captures result of `go test` or `gotestsum`
```

### Executing a compiled test binary

`gotestsum` supports executing a compiled test binary (created with `go test -c`) by running
Expand Down
55 changes: 48 additions & 7 deletions cmd/main.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cmd

import (
"bytes"
"context"
"errors"
"fmt"
Expand Down Expand Up @@ -70,6 +71,10 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
"use different icons, see help for options")
flags.BoolVar(&opts.rawCommand, "raw-command", false,
"don't prepend 'go test -json' to the 'go test' command")
flags.BoolVar(&opts.readStdin, "stdin", false,
"don't run any command, instead read go test stdout from stdin")
flags.IntVar(&opts.readStderrFD, "stderr", 0,
"read go test stderr from a certain `file descriptor` (only valid in combination with -stdin)")
flags.BoolVar(&opts.ignoreNonJSONOutputLines, "ignore-non-json-output-lines", false,
"write non-JSON 'go test' output lines to stderr instead of failing")
flags.Lookup("ignore-non-json-output-lines").Hidden = true
Expand Down Expand Up @@ -176,6 +181,8 @@ type options struct {
formatOptions testjson.FormatOptions
debug bool
rawCommand bool
readStdin bool
readStderrFD int
ignoreNonJSONOutputLines bool
jsonFile string
jsonFileTimingEvents string
Expand All @@ -198,6 +205,8 @@ type options struct {
version bool

// shims for testing
stdin io.Reader
fd3 io.Reader
stdout io.Writer
stderr io.Writer
}
Expand All @@ -212,6 +221,15 @@ func (o options) Validate() error {
return fmt.Errorf("-failfast can not be used with --rerun-fails " +
"because not all test cases will run")
}
if o.rawCommand && o.readStdin {
return errors.New("--stdin and --raw-command are mutually exclusive")
}
if o.readStdin && len(o.args) > 0 {
return fmt.Errorf("--stdin does not support additional arguments (%q)", o.args)
}
if o.readStderrFD > 0 && !o.readStdin {
return errors.New("--stderr depends on --stdin")
}
return nil
}

Expand Down Expand Up @@ -264,29 +282,52 @@ func run(opts *options) error {
return err
}

goTestProc, err := startGoTestFn(ctx, "", goTestCmdArgs(opts, rerunOpts{}))
if err != nil {
return err
}

handler, err := newEventHandler(opts)
if err != nil {
return err
}
defer handler.Close() // nolint: errcheck
cfg := testjson.ScanConfig{
Stdout: goTestProc.stdout,
Stderr: goTestProc.stderr,
Handler: handler,
Stop: cancel,
IgnoreNonJSONOutputLines: opts.ignoreNonJSONOutputLines,
}

var goTestProc *proc
if opts.readStdin {
cfg.Stdout = os.Stdin
if opts.stdin != nil {
cfg.Stdout = opts.stdin
}
if opts.readStderrFD > 0 {
if opts.readStderrFD == 3 && opts.fd3 != nil {
cfg.Stderr = opts.fd3
} else {
cfg.Stderr = os.NewFile(uintptr(opts.readStderrFD), fmt.Sprintf("go test stderr on fd %d", opts.stderr))
}
} else {
cfg.Stderr = bytes.NewReader(nil)
}
} else {
p, err := startGoTestFn(ctx, "", goTestCmdArgs(opts, rerunOpts{}))
if err != nil {
return err
}
goTestProc = p
cfg.Stdout = p.stdout
cfg.Stderr = p.stderr
}

exec, err := testjson.ScanTestOutput(cfg)
handler.Flush()
if err != nil {
return finishRun(opts, exec, err)
}

if opts.readStdin {
return finishRun(opts, exec, nil)
}

exitErr := goTestProc.cmd.Wait()
if signum := atomic.LoadInt32(&goTestProc.signal); signum != 0 {
return finishRun(opts, exec, exitError{num: signalExitCode + int(signum)})
Expand Down
101 changes: 101 additions & 0 deletions cmd/main_e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -275,3 +275,104 @@ func TestE2E_IgnoresWarnings(t *testing.T) {
)
golden.Assert(t, out, "e2e/expected/"+t.Name())
}

func TestE2E_StdinNoError(t *testing.T) {
t.Setenv("GITHUB_ACTIONS", "no")

flags, opts := setupFlags("gotestsum")
args := []string{
"--stdin",
"--format=testname",
}
assert.NilError(t, flags.Parse(args))
opts.args = flags.Args()

bufStdout := new(bytes.Buffer)
opts.stdout = bufStdout
bufStderr := new(bytes.Buffer)
opts.stderr = bufStderr

in := `{"Time":"2024-06-16T14:46:00.343974039+02:00","Action":"start","Package":"example.com/test"}
{"Time":"2024-06-16T14:46:00.378597503+02:00","Action":"run","Package":"example.com/test","Test":"TestSomething"}
{"Time":"2024-06-16T14:46:00.378798569+02:00","Action":"pass","Package":"example.com/test","Test":"TestSomething","Elapsed":0}
{"Time":"2024-06-16T14:46:00.404809796+02:00","Action":"pass","Package":"example.com/test","Elapsed":0.061}
`
opts.stdin = strings.NewReader(in)

err := run(opts)
assert.NilError(t, err)
out := text.ProcessLines(t, bufStdout,
text.OpRemoveSummaryLineElapsedTime,
text.OpRemoveTestElapsedTime,
filepath.ToSlash, // for windows
)
golden.Assert(t, out, "e2e/expected/"+t.Name())
}

func TestE2E_StdinFailure(t *testing.T) {
t.Setenv("GITHUB_ACTIONS", "no")

flags, opts := setupFlags("gotestsum")
args := []string{
"--stdin",
"--format=testname",
}
assert.NilError(t, flags.Parse(args))
opts.args = flags.Args()

bufStdout := new(bytes.Buffer)
opts.stdout = bufStdout
bufStderr := new(bytes.Buffer)
opts.stderr = bufStderr

in := `{"Time":"2024-06-16T14:46:00.343974039+02:00","Action":"start","Package":"example.com/test"}
{"Time":"2024-06-16T14:46:00.378597503+02:00","Action":"run","Package":"example.com/test","Test":"TestSomething"}
{"Time":"2024-06-16T14:46:00.378798569+02:00","Action":"fail","Package":"example.com/test","Test":"TestSomething","Elapsed":0}
{"Time":"2024-06-16T14:46:00.404809796+02:00","Action":"pass","Package":"example.com/test","Elapsed":0.061}
`
opts.stdin = strings.NewReader(in)

err := run(opts)
assert.NilError(t, err)
out := text.ProcessLines(t, bufStdout,
text.OpRemoveSummaryLineElapsedTime,
text.OpRemoveTestElapsedTime,
filepath.ToSlash, // for windows
)
golden.Assert(t, out, "e2e/expected/"+t.Name())
}

func TestE2E_StdinStderr(t *testing.T) {
t.Setenv("GITHUB_ACTIONS", "no")

flags, opts := setupFlags("gotestsum")
args := []string{
"--stdin",
"--stderr=3",
"--format=testname",
}
assert.NilError(t, flags.Parse(args))
opts.args = flags.Args()

bufStdout := new(bytes.Buffer)
opts.stdout = bufStdout
bufStderr := new(bytes.Buffer)
opts.stderr = bufStderr

in := `{"Time":"2024-06-16T14:46:00.343974039+02:00","Action":"start","Package":"example.com/test"}
{"Time":"2024-06-16T14:46:00.378597503+02:00","Action":"run","Package":"example.com/test","Test":"TestSomething"}
{"Time":"2024-06-16T14:46:00.378798569+02:00","Action":"pass","Package":"example.com/test","Test":"TestSomething","Elapsed":0}
{"Time":"2024-06-16T14:46:00.404809796+02:00","Action":"pass","Package":"example.com/test","Elapsed":0.061}
`
opts.stdin = strings.NewReader(in)
opts.fd3 = strings.NewReader(`build failure`)

err := run(opts)
assert.NilError(t, err)
out := text.ProcessLines(t, bufStdout,
text.OpRemoveSummaryLineElapsedTime,
text.OpRemoveTestElapsedTime,
filepath.ToSlash, // for windows
)
golden.Assert(t, out, "e2e/expected/"+t.Name())
}
15 changes: 15 additions & 0 deletions cmd/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,21 @@ func TestOptions_Validate_FromFlags(t *testing.T) {
args: []string{"--rerun-fails", "--packages=./...", "--", "-failfast"},
expected: "-failfast can not be used with --rerun-fails",
},
{
name: "raw-command and stdin mutually exclusive",
args: []string{"--raw-command", "--stdin"},
expected: "--stdin and --raw-command are mutually exclusive",
},
{
name: "stdin must not be used with args",
args: []string{"--stdin", "--", "-coverprofile=/tmp/out"},
expected: `--stdin does not support additional arguments (["-coverprofile=/tmp/out"])`,
},
{
name: "stderr depends on stdin",
args: []string{"--stderr", "4"},
expected: "--stderr depends on --stdin",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
Expand Down
7 changes: 7 additions & 0 deletions cmd/testdata/e2e/expected/TestE2E_StdinFailure
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
FAIL example.com/test.TestSomething
PASS example.com/test

=== Failed
=== FAIL: example.com/test TestSomething

DONE 1 tests, 1 failure
4 changes: 4 additions & 0 deletions cmd/testdata/e2e/expected/TestE2E_StdinNoError
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
PASS example.com/test.TestSomething
PASS example.com/test

DONE 1 tests
7 changes: 7 additions & 0 deletions cmd/testdata/e2e/expected/TestE2E_StdinStderr
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
PASS example.com/test.TestSomething
PASS example.com/test

=== Errors
build failure

DONE 1 tests, 1 error
2 changes: 2 additions & 0 deletions cmd/testdata/gotestsum-help-text
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ Flags:
--rerun-fails-max-failures int do not rerun any tests if the initial run has more than this number of failures (default 10)
--rerun-fails-report string write a report to the file, of the tests that were rerun
--rerun-fails-run-root-test rerun the entire root testcase when any of its subtests fail, instead of only the failed subtest
--stderr file descriptor read go test stderr from a certain file descriptor (only valid in combination with -stdin)
--stdin don't run any command, instead read go test stdout from stdin
--version show version and exit
--watch watch go files, and run tests when a file is modified
--watch-chdir in watch mode change the working directory to the directory with the modified file before running tests
Expand Down

0 comments on commit 8475e03

Please sign in to comment.