Merge pull request #4 from fred1268/release-1.2.0

Release 1.2.0

README.md

@@ -155,6 +155,7 @@ A test file looks like the following:
       "server": "hackernews",
       "method": "POST",
       "endpoint": "/v0/item",
+      "urlparams": "{\"q\":\"search terms\"}",
       "payload": "@custom_filename.json",
       "expected": {
         "statuscode": 200
@@ -186,10 +187,14 @@ A test file contains an array of tests, each of them containing:
 
 - `endpoint` (mandatory): the endpoint of the operation (usually a ReST API of some sort)
 
+- `urlparams` (default none): an object whose keys/values represents URL parameters' keys and values
+
 - `capture` (default false): true if you want to capture the response of this test so that it can be used in another test in this file (fileParallel mode only)
 
 - `skip` (default false): true to have okapi skip this test (useful when debugging a script file)
 
+- `debug` (default false): true to have okapi display debugging information (see debugging tests below)
+
 - `payload` (default none): the payload to be sent to the endpoint (usually with a POST, PUT or PATCH method)
 
 - `expected`: this section contains:
@@ -245,6 +250,8 @@ where options are one or more of the following:
 
 - `--no-parallel` (default parallel): prevent tests from running in parallel
 
+- `--workers` (default #cores): define the maximum number of workers
+
 - `--user-agent` (default okapi UA): set the default user agent
 
 - `--content-type` (default application/json): set the default content type for requests
@@ -292,6 +299,26 @@ ok      hackernews.users.test.json                      0.368s
 okapi total run time: 0.368s
 ```
 
+## Debugging tests
+
+Writing tests is tedious and it can be pretty difficult to understand what is going on in a test within a full test file running in parallel. In order to help debug your tests, okapi provides a few mechanisms.
+
+### okapi header
+
+With each HTTP request, okapi will send its user agent of course (default `Mozilla/5.0 (compatible; okapi/1.x; +https://github.com/fred1268/okapi)`), but also a special header, `X-okapi-testname` which will contain the name of the test. You can use this information on the server side for instance to display a banner in your logs to delimit the start and end of the tests, making it easier to find the logs corresponding to a particular test.
+
+### okapi run modes
+
+Running tests in `--file-parallel` or `--no-parallel` mode can make it easier to troubleshoot tests, since the tests output, mainly on the server side, won't overlap. Also, using the `--file` or the `--test` to work on a specific file or test respectively will also make this process smoother.
+
+### Test debug flag
+
+Each test can also have its `debug` flag set to true. This tells okapi to display detailed information about the HTTP request being made to the server, including URL, Method, Payload, Parameters, Header, etc. This can be helpful to verify the good setup of your tests, including, but not limited to, captured information.
+
+### Test skip flag
+
+Finally, do not forget that you can skip some tests with the `skip` flag to reduce the amount of information displayed both on the okapi side but also on your server's logs.
+
 ## Integrating okapi :giraffe: with your own software
 
 okapi exposes a pretty simple and straightforward API that you can use within your own Go programs.

main.go

@@ -19,17 +19,23 @@ func help() {
 	fmt.Println("The options are:")
 	fmt.Println()
 	fmt.Println("\t--servers-file, -s (mandatory):\t\t\t\tpoint to the configuration file's location")
-	fmt.Println("\t--timeout (default 30s):\t\t\t\tset a default timeout for all HTTP requests")
 	fmt.Println("\t--verbose, -v (default no):\t\t\t\tenable verbose mode")
+	fmt.Println("\t--file-parallel (default no):\t\t\t\trun the test files in parallel (instead of the tests themselves)")
+	fmt.Println("\t--file, -f (default none):\t\t\t\tonly run the specified test file")
+	fmt.Println("\t--test, -t (default none):\t\t\t\tonly run the specified standalone test")
+	fmt.Println("\t--timeout (default 30s):\t\t\t\tset a default timeout for all HTTP requests")
 	fmt.Println("\t--no-parallel (default parallel):\t\t\tprevent tests from running in parallel")
+	fmt.Println("\t--workers (default #cores):\t\t\t\tdefine the maximum number of workers")
 	fmt.Println("\t--user-agent (default okapi UA):\t\t\tset the default user agent")
-	fmt.Println("\t--content-type (default application/json):\t\tset the default content type for requests")
-	fmt.Println("\t--accept (default application/json):\t\t\tset the default accept header for responses")
+	fmt.Println("\t--content-type (default 'application/json'):\t\tset the default content type for requests")
+	fmt.Println("\t--accept (default 'application/json'):\t\t\tset the default accept header for responses")
 	fmt.Println()
 	fmt.Println("The parameters are:")
 	fmt.Println()
 	fmt.Println("\ttest_directory:\t\t\t\t\t\tpoint to the directory where all the test files are located")
 	fmt.Println()
+	fmt.Println("More information (and source code) on: https://github.com/fred1268/okapi")
+	fmt.Println()
 }
 
 func main() {

testing/api.go

@@ -42,6 +42,10 @@ type APIRequest struct {
 	// when debugging script files or to allow tests to
 	// pass while a bug is being fixed for instance.
 	Skip bool
+	// Debug will make okapi output test debugging
+	// information to ease troubleshooting errors
+	Debug  bool
+	atFile bool
 }
 
 // APIResponse contains information about the response from
@@ -55,7 +59,8 @@ type APIResponse struct {
 	Response string
 	// Logs represents okapi's logs which are grouped later
 	// on to be nicely displayed even in parallel mode.
-	Logs []string
+	Logs   []string
+	atFile bool
 }
 
 func (a *APIRequest) validate() error {
@@ -72,3 +77,7 @@ func (a *APIRequest) validate() error {
 	a.Payload = os.SubstituteEnvironmentVariable(a.Payload)
 	return nil
 }
+
+func (a *APIRequest) hasFileDepencies() bool {
+	return a.atFile || a.Expected.atFile
+}

testing/client.go

@@ -73,43 +73,69 @@ func (c *Client) buildEndpointURL(ctx context.Context, apiRequest *APIRequest) (
 	return addr, nil
 }
 
-func (c *Client) getRequest(ctx context.Context, apiRequest *APIRequest) (*http.Request, error) {
+func (c *Client) getRequest(ctx context.Context, apiRequest *APIRequest, apiResponse *APIResponse) (*http.Request, error) {
 	addr, err := c.buildEndpointURL(ctx, apiRequest)
 	if err != nil {
 		return nil, err
 	}
+	if apiRequest.Debug {
+		apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf("--- %s\n", apiRequest.Name))
+		apiResponse.Logs = append(apiResponse.Logs, "API Request:\n")
+		apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf("  URL: %s\n", addr))
+		apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf("  Method: %s\n", apiRequest.Method))
+		apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf("  Payload: %s\n", apiRequest.Payload))
+		apiResponse.Logs = append(apiResponse.Logs, "  Headers:\n")
+	}
 	req, err := http.NewRequestWithContext(ctx, strings.ToUpper(apiRequest.Method), addr,
 		bytes.NewBufferString(apiRequest.Payload))
 	if err != nil {
 		return nil, err
 	}
 	if c.config.Auth != nil && c.config.Auth.APIKey != nil {
+		if apiRequest.Debug {
+			apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf("    %s: %s\n", c.config.Auth.APIKey.Header,
+				c.config.Auth.APIKey.APIKey))
+		}
 		req.Header.Set(c.config.Auth.APIKey.Header, c.config.Auth.APIKey.APIKey)
 	}
 	if c.config.UserAgent != "" {
+		if apiRequest.Debug {
+			apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf("    User-Agent: %s\n", c.config.UserAgent))
+		}
 		req.Header.Add("User-Agent", c.config.UserAgent)
 	}
 	if c.jwt != "" {
+		if apiRequest.Debug {
+			apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf("    Authorization: Bearer: %s\n", c.jwt))
+		}
 		req.Header.Set("authorization", fmt.Sprintf("Bearer: %s", c.jwt))
 	}
 	if c.cookie != nil {
 		req.AddCookie(c.cookie)
 	}
+	req.Header.Add("X-okapi-testname", apiRequest.Name)
 	if len(apiRequest.Headers) != 0 {
 		for key, value := range apiRequest.Headers {
+			if apiRequest.Debug {
+				apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf("    %s: %s\n", key, value))
+			}
 			req.Header.Add(key, value)
 		}
 	} else {
 		for key, value := range c.config.Headers {
+			if apiRequest.Debug {
+				apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf("    %s: %s\n", key, value))
+			}
 			req.Header.Add(key, value)
 		}
 	}
 	return req, nil
 }
 
-func (c *Client) call(ctx context.Context, apiRequest *APIRequest) (response *APIResponse, err error) {
+func (c *Client) call(ctx context.Context, apiRequest *APIRequest) (apiResponse *APIResponse, err error) {
+	apiResponse = &APIResponse{}
 	var req *http.Request
-	req, err = c.getRequest(ctx, apiRequest)
+	req, err = c.getRequest(ctx, apiRequest, apiResponse)
 	if err != nil {
 		return
 	}
@@ -135,6 +161,10 @@ func (c *Client) call(ctx context.Context, apiRequest *APIRequest) (response *AP
 	if err != nil {
 		return
 	}
+	if apiRequest.Debug {
+		apiResponse.Logs = append(apiResponse.Logs, "API Response:\n")
+		apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf("  Response: %s", string(res)))
+	}
 	if c.jwt == "" && c.config.Auth != nil && c.config.Auth.Session != nil && c.config.Auth.Session.JWT != "" {
 		switch c.config.Auth.Session.JWT {
 		case "payload":
@@ -155,7 +185,8 @@ func (c *Client) call(ctx context.Context, apiRequest *APIRequest) (response *AP
 			}
 		}
 	}
-	response = &APIResponse{StatusCode: resp.StatusCode, Response: string(res)}
+	apiResponse.StatusCode = resp.StatusCode
+	apiResponse.Response = string(res)
 	return
 }
 

testing/config.go

@@ -1,6 +1,10 @@
 package testing
 
-import "github.com/fred1268/go-clap/clap"
+import (
+	"runtime"
+
+	"github.com/fred1268/go-clap/clap"
+)
 
 // Config holds okapi's configuration.
 type Config struct {
@@ -10,11 +14,12 @@ type Config struct {
 	UserAgent    string `clap:"--user-agent"`
 	ContentType  string `clap:"--content-type"`
 	Accept       string `clap:"--accept"`
+	File         string `clap:"--file,-f"`
+	Test         string `clap:"--test,-t"`
+	Workers      int    `clap:"--workers"`
 	Verbose      bool   `clap:"--verbose,-v"`
 	Parallel     bool   `clap:"--parallel,-p"`
 	FileParallel bool   `clap:"--file-parallel"`
-	File         string `clap:"--file,-f"`
-	Test         string `clap:"--test,-t"`
 }
 
 // LoadConfig returns okapi's configuration from the
@@ -23,12 +28,12 @@ type Config struct {
 // see: https://github.com/fred1268/go-clap
 func LoadConfig(args []string) (*Config, error) {
 	var cfg Config = Config{
-		Timeout:      30,
-		UserAgent:    "Mozilla/5.0 (compatible; okapi/1.0; +https://github.com/fred1268/okapi)",
-		ContentType:  "application/json",
-		Accept:       "application/json",
-		Parallel:     true,
-		FileParallel: false,
+		Timeout:     30,
+		UserAgent:   "Mozilla/5.0 (compatible; okapi/1.x; +https://github.com/fred1268/okapi)",
+		ContentType: "application/json",
+		Accept:      "application/json",
+		Workers:     runtime.NumCPU(),
+		Parallel:    true,
 	}
 	if _, err := clap.Parse(args, &cfg); err != nil {
 		return nil, err

testing/loader.go

@@ -61,6 +61,7 @@ func LoadTests(cfg *Config) (map[string][]*APIRequest, error) {
 	if err != nil {
 		return nil, err
 	}
+	uniqueTests := make(map[string]*APIRequest)
 	allTests := make(map[string][]*APIRequest)
 	for _, file := range files {
 		if !strings.HasSuffix(file.Name(), ".test.json") {
@@ -79,6 +80,31 @@ func LoadTests(cfg *Config) (map[string][]*APIRequest, error) {
 		if err = json.NewDecoder(bytes.NewReader(content)).Decode(&tests); err != nil {
 			return nil, fmt.Errorf("cannot decode json file '%s': %w", file.Name(), err)
 		}
+		for _, test := range tests.Tests {
+			if test.Payload == "@file" {
+				test.atFile = true
+			}
+			if test.Expected.Response == "@file" {
+				test.Expected.atFile = true
+			}
+		}
+		for _, test := range tests.Tests {
+			t, ok := uniqueTests[test.Name]
+			if !ok {
+				uniqueTests[test.Name] = test
+				continue
+			}
+			log.Printf("Warning: two tests with the same name (%s)\n", test.Name)
+			if t.hasFileDepencies() {
+				if test.hasFileDepencies() {
+					log.Printf("Potential conflict: two tests with the same name (%s) using @file\n", test.Name)
+				}
+			} else {
+				// replace test without @file with this one
+				// doesn't matter if it has @file or not
+				uniqueTests[test.Name] = test
+			}
+		}
 		if err := readJSONDependencies(cfg.Directory, tests.Tests); err != nil {
 			return nil, err
 		}

testing/test.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"runtime"
 	"strings"
 	"sync"
 	"time"
@@ -114,7 +113,7 @@ func printer(ctx context.Context, allTests map[string][]*APIRequest, out chan *t
 				log.Printf("FAIL\t%s\t\t\t%0.3fs\n", tout.file, time.Since(tout.fileStart).Seconds())
 				log.Printf("FAIL \n")
 			} else {
-				log.Printf("ok\t%-30s\t\t\t%0.3fs\n", tout.file, time.Since(tout.fileStart).Seconds())
+				log.Printf("ok\t%-45s\t\t%0.3fs\n", fmt.Sprintf("%s (%d tests)", tout.file, len(allTests[tout.file])), time.Since(tout.fileStart).Seconds())
 			}
 			files++
 		}
@@ -146,11 +145,11 @@ func Run(ctx context.Context, cfg *Config) error {
 	in := make(chan []*testIn)
 	done := make(chan bool)
 	var wg sync.WaitGroup
-	cpu := 1
+	workers := 1
 	if cfg.Parallel || cfg.FileParallel {
-		cpu = runtime.NumCPU()
+		workers = cfg.Workers
 	}
-	for i := 0; i < cpu; i++ {
+	for i := 0; i < workers; i++ {
 		go worker(ctx, in, out, done)
 	}
 	wg.Add(1)
@@ -185,6 +184,10 @@ func Run(ctx context.Context, cfg *Config) error {
 	close(done)
 	close(in)
 	close(out)
-	log.Printf("okapi total run time: %0.3fs\n", time.Since(start).Seconds())
+	count := 0
+	for _, value := range allTests {
+		count += len(value)
+	}
+	log.Printf("okapi total run time: %0.3fs (%d tests total)\n", time.Since(start).Seconds(), count)
 	return nil
 }