From bc418fbc4e2447c3f9c521ab2cd9034d1866278d Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 1 Aug 2023 15:15:21 +0200 Subject: [PATCH 01/12] Display the number of tetss --- testing/test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/testing/test.go b/testing/test.go index f977ab2..131b770 100644 --- a/testing/test.go +++ b/testing/test.go @@ -114,7 +114,7 @@ func printer(ctx context.Context, allTests map[string][]*APIRequest, out chan *t log.Printf("FAIL\t%s\t\t\t%0.3fs\n", tout.file, time.Since(tout.fileStart).Seconds()) log.Printf("FAIL \n") } else { - log.Printf("ok\t%-30s\t\t\t%0.3fs\n", tout.file, time.Since(tout.fileStart).Seconds()) + log.Printf("ok\t%-45s\t\t%0.3fs\n", fmt.Sprintf("%s (%d tests)", tout.file, len(allTests[tout.file])), time.Since(tout.fileStart).Seconds()) } files++ } @@ -185,6 +185,10 @@ func Run(ctx context.Context, cfg *Config) error { close(done) close(in) close(out) - log.Printf("okapi total run time: %0.3fs\n", time.Since(start).Seconds()) + count := 0 + for _, value := range allTests { + count += len(value) + } + log.Printf("okapi total run time: %0.3fs (%d tests total)\n", time.Since(start).Seconds(), count) return nil } From a164dc38d80aa3e5a62d63d21db947ca822166d8 Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 1 Aug 2023 15:16:05 +0200 Subject: [PATCH 02/12] Add flag to chose the number of CPUs to use --- testing/config.go | 12 +++++++++--- testing/test.go | 3 +-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/testing/config.go b/testing/config.go index 9d3dfc1..9c06e2a 100644 --- a/testing/config.go +++ b/testing/config.go @@ -1,6 +1,10 @@ package testing -import "github.com/fred1268/go-clap/clap" +import ( + "runtime" + + "github.com/fred1268/go-clap/clap" +) // Config holds okapi's configuration. type Config struct { @@ -10,11 +14,12 @@ type Config struct { UserAgent string `clap:"--user-agent"` ContentType string `clap:"--content-type"` Accept string `clap:"--accept"` + File string `clap:"--file,-f"` + Test string `clap:"--test,-t"` + CPU int `clap:"--cpu"` Verbose bool `clap:"--verbose,-v"` Parallel bool `clap:"--parallel,-p"` FileParallel bool `clap:"--file-parallel"` - File string `clap:"--file,-f"` - Test string `clap:"--test,-t"` } // LoadConfig returns okapi's configuration from the @@ -27,6 +32,7 @@ func LoadConfig(args []string) (*Config, error) { UserAgent: "Mozilla/5.0 (compatible; okapi/1.0; +https://github.com/fred1268/okapi)", ContentType: "application/json", Accept: "application/json", + CPU: runtime.NumCPU(), Parallel: true, FileParallel: false, } diff --git a/testing/test.go b/testing/test.go index 131b770..5dc3c9f 100644 --- a/testing/test.go +++ b/testing/test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "runtime" "strings" "sync" "time" @@ -148,7 +147,7 @@ func Run(ctx context.Context, cfg *Config) error { var wg sync.WaitGroup cpu := 1 if cfg.Parallel || cfg.FileParallel { - cpu = runtime.NumCPU() + cpu = cfg.CPU } for i := 0; i < cpu; i++ { go worker(ctx, in, out, done) From 1d0edd5a559dea7659c4b1e5603ae53a2bcfa6dc Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 1 Aug 2023 15:19:07 +0200 Subject: [PATCH 03/12] Rename cpu to worker to reflect reality --- testing/config.go | 4 ++-- testing/test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/testing/config.go b/testing/config.go index 9c06e2a..ce01e7a 100644 --- a/testing/config.go +++ b/testing/config.go @@ -16,7 +16,7 @@ type Config struct { Accept string `clap:"--accept"` File string `clap:"--file,-f"` Test string `clap:"--test,-t"` - CPU int `clap:"--cpu"` + Workers int `clap:"--workers"` Verbose bool `clap:"--verbose,-v"` Parallel bool `clap:"--parallel,-p"` FileParallel bool `clap:"--file-parallel"` @@ -32,7 +32,7 @@ func LoadConfig(args []string) (*Config, error) { UserAgent: "Mozilla/5.0 (compatible; okapi/1.0; +https://github.com/fred1268/okapi)", ContentType: "application/json", Accept: "application/json", - CPU: runtime.NumCPU(), + Workers: runtime.NumCPU(), Parallel: true, FileParallel: false, } diff --git a/testing/test.go b/testing/test.go index 5dc3c9f..ceff97d 100644 --- a/testing/test.go +++ b/testing/test.go @@ -145,11 +145,11 @@ func Run(ctx context.Context, cfg *Config) error { in := make(chan []*testIn) done := make(chan bool) var wg sync.WaitGroup - cpu := 1 + workers := 1 if cfg.Parallel || cfg.FileParallel { - cpu = cfg.CPU + workers = cfg.Workers } - for i := 0; i < cpu; i++ { + for i := 0; i < workers; i++ { go worker(ctx, in, out, done) } wg.Add(1) From ad2bfa522950bfff80ce2f3eb9b9f9aca7f7cb57 Mon Sep 17 00:00:00 2001 From: Fred Date: Wed, 2 Aug 2023 20:50:45 +0200 Subject: [PATCH 04/12] Add documentation for URLParams --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index f752317..823664c 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ A test file looks like the following: "server": "hackernews", "method": "POST", "endpoint": "/v0/item", + "urlparams": "{\"q\":\"search terms\"}" "payload": "@custom_filename.json", "expected": { "statuscode": 200 @@ -186,6 +187,8 @@ A test file contains an array of tests, each of them containing: - `endpoint` (mandatory): the endpoint of the operation (usually a ReST API of some sort) +- `urlparams` (default none): an object whose keys/values represents URL parameters' keys and values + - `capture` (default false): true if you want to capture the response of this test so that it can be used in another test in this file (fileParallel mode only) - `skip` (default false): true to have okapi skip this test (useful when debugging a script file) From 0550cd1ed1528a2a7231bbad0d0f3c98874ac09c Mon Sep 17 00:00:00 2001 From: Fred Date: Wed, 2 Aug 2023 20:51:06 +0200 Subject: [PATCH 05/12] Check for tests names duplication --- testing/api.go | 10 ++++++++-- testing/loader.go | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/testing/api.go b/testing/api.go index 1796242..1ba64a3 100644 --- a/testing/api.go +++ b/testing/api.go @@ -41,7 +41,8 @@ type APIRequest struct { // Skip will make okapi skip this test. Can use useful // when debugging script files or to allow tests to // pass while a bug is being fixed for instance. - Skip bool + Skip bool + atFile bool } // APIResponse contains information about the response from @@ -55,7 +56,8 @@ type APIResponse struct { Response string // Logs represents okapi's logs which are grouped later // on to be nicely displayed even in parallel mode. - Logs []string + Logs []string + atFile bool } func (a *APIRequest) validate() error { @@ -72,3 +74,7 @@ func (a *APIRequest) validate() error { a.Payload = os.SubstituteEnvironmentVariable(a.Payload) return nil } + +func (a *APIRequest) hasFileDepencies() bool { + return a.atFile || a.Expected.atFile +} diff --git a/testing/loader.go b/testing/loader.go index d687c5c..57b35f2 100644 --- a/testing/loader.go +++ b/testing/loader.go @@ -61,6 +61,7 @@ func LoadTests(cfg *Config) (map[string][]*APIRequest, error) { if err != nil { return nil, err } + uniqueTests := make(map[string]*APIRequest) allTests := make(map[string][]*APIRequest) for _, file := range files { if !strings.HasSuffix(file.Name(), ".test.json") { @@ -79,6 +80,31 @@ func LoadTests(cfg *Config) (map[string][]*APIRequest, error) { if err = json.NewDecoder(bytes.NewReader(content)).Decode(&tests); err != nil { return nil, fmt.Errorf("cannot decode json file '%s': %w", file.Name(), err) } + for _, test := range tests.Tests { + if test.Payload == "@file" { + test.atFile = true + } + if test.Expected.Response == "@file" { + test.Expected.atFile = true + } + } + for _, test := range tests.Tests { + t, ok := uniqueTests[test.Name] + if !ok { + uniqueTests[test.Name] = test + continue + } + log.Printf("Warning: two tests with the same name (%s)\n", test.Name) + if t.hasFileDepencies() { + if test.hasFileDepencies() { + log.Printf("Potential conflict: two tests with the same name (%s) using @file\n", test.Name) + } + } else { + // replace test without @file with this one + // doesn't matter if it has @file or not + uniqueTests[test.Name] = test + } + } if err := readJSONDependencies(cfg.Directory, tests.Tests); err != nil { return nil, err } From 6f0e6e054ee4cdb0a0fc7c784a12f8422964cbe9 Mon Sep 17 00:00:00 2001 From: Fred Date: Wed, 2 Aug 2023 21:44:13 +0200 Subject: [PATCH 06/12] Add support for okapi header --- testing/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/client.go b/testing/client.go index 99d1ad0..60b4db5 100644 --- a/testing/client.go +++ b/testing/client.go @@ -95,6 +95,7 @@ func (c *Client) getRequest(ctx context.Context, apiRequest *APIRequest) (*http. if c.cookie != nil { req.AddCookie(c.cookie) } + req.Header.Add("x-okapi-testname", apiRequest.Name) if len(apiRequest.Headers) != 0 { for key, value := range apiRequest.Headers { req.Header.Add(key, value) From 8fba760cf98ee7accdd431345fe756b103766fd6 Mon Sep 17 00:00:00 2001 From: Fred Date: Thu, 3 Aug 2023 08:50:08 +0200 Subject: [PATCH 07/12] Remove useless initialization --- testing/config.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/testing/config.go b/testing/config.go index ce01e7a..c372eca 100644 --- a/testing/config.go +++ b/testing/config.go @@ -28,13 +28,12 @@ type Config struct { // see: https://github.com/fred1268/go-clap func LoadConfig(args []string) (*Config, error) { var cfg Config = Config{ - Timeout: 30, - UserAgent: "Mozilla/5.0 (compatible; okapi/1.0; +https://github.com/fred1268/okapi)", - ContentType: "application/json", - Accept: "application/json", - Workers: runtime.NumCPU(), - Parallel: true, - FileParallel: false, + Timeout: 30, + UserAgent: "Mozilla/5.0 (compatible; okapi/1.0; +https://github.com/fred1268/okapi)", + ContentType: "application/json", + Accept: "application/json", + Workers: runtime.NumCPU(), + Parallel: true, } if _, err := clap.Parse(args, &cfg); err != nil { return nil, err From 3c083b4d231dc182beda696ee3ce7b2b9598f427 Mon Sep 17 00:00:00 2001 From: Fred Date: Thu, 3 Aug 2023 15:24:28 +0200 Subject: [PATCH 08/12] Make okapi version more generic --- testing/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/config.go b/testing/config.go index c372eca..2dae78a 100644 --- a/testing/config.go +++ b/testing/config.go @@ -29,7 +29,7 @@ type Config struct { func LoadConfig(args []string) (*Config, error) { var cfg Config = Config{ Timeout: 30, - UserAgent: "Mozilla/5.0 (compatible; okapi/1.0; +https://github.com/fred1268/okapi)", + UserAgent: "Mozilla/5.0 (compatible; okapi/1.x; +https://github.com/fred1268/okapi)", ContentType: "application/json", Accept: "application/json", Workers: runtime.NumCPU(), From 68c3af6c146d90bd33ba30566642046034a4a94a Mon Sep 17 00:00:00 2001 From: Fred Date: Thu, 3 Aug 2023 15:32:44 +0200 Subject: [PATCH 09/12] Add test debugging documentation and support --- README.md | 24 +++++++++++++++++++++++- testing/api.go | 5 ++++- testing/client.go | 40 +++++++++++++++++++++++++++++++++++----- 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 823664c..17b01e6 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ A test file looks like the following: "server": "hackernews", "method": "POST", "endpoint": "/v0/item", - "urlparams": "{\"q\":\"search terms\"}" + "urlparams": "{\"q\":\"search terms\"}", "payload": "@custom_filename.json", "expected": { "statuscode": 200 @@ -193,6 +193,8 @@ A test file contains an array of tests, each of them containing: - `skip` (default false): true to have okapi skip this test (useful when debugging a script file) +- `debug` (default false): true to have okapi display debugging information (see debugging tests below) + - `payload` (default none): the payload to be sent to the endpoint (usually with a POST, PUT or PATCH method) - `expected`: this section contains: @@ -295,6 +297,26 @@ ok hackernews.users.test.json 0.368s okapi total run time: 0.368s ``` +## Debugging tests + +Writing tests is tedious and it can be pretty difficult to understand what is going on in a test within a full test file running in parallel. In order to help debug your tests, okapi provides a few mechanisms. + +### okapi header + +With each HTTP request, okapi will send its user agent of course (default `Mozilla/5.0 (compatible; okapi/1.x; +https://github.com/fred1268/okapi)`), but also a special header, `X-okapi-testname` which will contain the name of the test. You can use this information on the server side for instance to display a banner in your logs to delimit the start and end of the tests, making it easier to find the logs corresponding to a particular test. + +### okapi run modes + +Running tests in `--file-parallel` or `--no-parallel` mode can make it easier to troubleshoot tests, since the tests output, mainly on the server side, won't overlap. Also, using the `--file` or the `--test` to work on a specific file or test respectively will also make this process smoother. + +### Test debug flag + +Each test can also have its `debug` flag set to true. This tells okapi to display detailed information about the HTTP request being made to the server, including URL, Method, Payload, Parameters, Header, etc. This can be helpful to verify the good setup of your tests, including, but not limited to, captured information. + +### Test skip flag + +Finally, do not forget that you can skip some tests with the `skip` flag to reduce the amount of information displayed both on the okapi side but also on your server's logs. + ## Integrating okapi :giraffe: with your own software okapi exposes a pretty simple and straightforward API that you can use within your own Go programs. diff --git a/testing/api.go b/testing/api.go index 1ba64a3..9835bd9 100644 --- a/testing/api.go +++ b/testing/api.go @@ -41,7 +41,10 @@ type APIRequest struct { // Skip will make okapi skip this test. Can use useful // when debugging script files or to allow tests to // pass while a bug is being fixed for instance. - Skip bool + Skip bool + // Debug will make okapi output test debugging + // information to ease troubleshooting errors + Debug bool atFile bool } diff --git a/testing/client.go b/testing/client.go index 60b4db5..409680a 100644 --- a/testing/client.go +++ b/testing/client.go @@ -73,44 +73,69 @@ func (c *Client) buildEndpointURL(ctx context.Context, apiRequest *APIRequest) ( return addr, nil } -func (c *Client) getRequest(ctx context.Context, apiRequest *APIRequest) (*http.Request, error) { +func (c *Client) getRequest(ctx context.Context, apiRequest *APIRequest, apiResponse *APIResponse) (*http.Request, error) { addr, err := c.buildEndpointURL(ctx, apiRequest) if err != nil { return nil, err } + if apiRequest.Debug { + apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf("--- %s\n", apiRequest.Name)) + apiResponse.Logs = append(apiResponse.Logs, "API Request:\n") + apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf(" URL: %s\n", addr)) + apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf(" Method: %s\n", apiRequest.Method)) + apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf(" Payload: %s\n", apiRequest.Payload)) + apiResponse.Logs = append(apiResponse.Logs, " Headers:\n") + } req, err := http.NewRequestWithContext(ctx, strings.ToUpper(apiRequest.Method), addr, bytes.NewBufferString(apiRequest.Payload)) if err != nil { return nil, err } if c.config.Auth != nil && c.config.Auth.APIKey != nil { + if apiRequest.Debug { + apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf(" %s: %s\n", c.config.Auth.APIKey.Header, + c.config.Auth.APIKey.APIKey)) + } req.Header.Set(c.config.Auth.APIKey.Header, c.config.Auth.APIKey.APIKey) } if c.config.UserAgent != "" { + if apiRequest.Debug { + apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf(" User-Agent: %s\n", c.config.UserAgent)) + } req.Header.Add("User-Agent", c.config.UserAgent) } if c.jwt != "" { + if apiRequest.Debug { + apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf(" Authorization: Bearer: %s\n", c.jwt)) + } req.Header.Set("authorization", fmt.Sprintf("Bearer: %s", c.jwt)) } if c.cookie != nil { req.AddCookie(c.cookie) } - req.Header.Add("x-okapi-testname", apiRequest.Name) + req.Header.Add("X-okapi-testname", apiRequest.Name) if len(apiRequest.Headers) != 0 { for key, value := range apiRequest.Headers { + if apiRequest.Debug { + apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf(" %s: %s\n", key, value)) + } req.Header.Add(key, value) } } else { for key, value := range c.config.Headers { + if apiRequest.Debug { + apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf(" %s: %s\n", key, value)) + } req.Header.Add(key, value) } } return req, nil } -func (c *Client) call(ctx context.Context, apiRequest *APIRequest) (response *APIResponse, err error) { +func (c *Client) call(ctx context.Context, apiRequest *APIRequest) (apiResponse *APIResponse, err error) { + apiResponse = &APIResponse{} var req *http.Request - req, err = c.getRequest(ctx, apiRequest) + req, err = c.getRequest(ctx, apiRequest, apiResponse) if err != nil { return } @@ -136,6 +161,10 @@ func (c *Client) call(ctx context.Context, apiRequest *APIRequest) (response *AP if err != nil { return } + if apiRequest.Debug { + apiResponse.Logs = append(apiResponse.Logs, "API Response:\n") + apiResponse.Logs = append(apiResponse.Logs, fmt.Sprintf(" Response: %s", string(res))) + } if c.jwt == "" && c.config.Auth != nil && c.config.Auth.Session != nil && c.config.Auth.Session.JWT != "" { switch c.config.Auth.Session.JWT { case "payload": @@ -156,7 +185,8 @@ func (c *Client) call(ctx context.Context, apiRequest *APIRequest) (response *AP } } } - response = &APIResponse{StatusCode: resp.StatusCode, Response: string(res)} + apiResponse.StatusCode = resp.StatusCode + apiResponse.Response = string(res) return } From 2a9fc3e0c12e1323b65f71c3a2aeae07994585cb Mon Sep 17 00:00:00 2001 From: Fred Date: Thu, 3 Aug 2023 15:40:52 +0200 Subject: [PATCH 10/12] Update help command --- main.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index e7ed1eb..f680318 100644 --- a/main.go +++ b/main.go @@ -19,12 +19,15 @@ func help() { fmt.Println("The options are:") fmt.Println() fmt.Println("\t--servers-file, -s (mandatory):\t\t\t\tpoint to the configuration file's location") - fmt.Println("\t--timeout (default 30s):\t\t\t\tset a default timeout for all HTTP requests") fmt.Println("\t--verbose, -v (default no):\t\t\t\tenable verbose mode") + fmt.Println("\t--file-parallel (default no):\t\t\t\trun the test files in parallel (instead of the tests themselves)") + fmt.Println("\t--file, -f (default none):\t\t\t\tonly run the specified test file") + fmt.Println("\t--test, -t (default none):\t\t\t\tonly run the specified standalone test") + fmt.Println("\t--timeout (default 30s):\t\t\t\tset a default timeout for all HTTP requests") fmt.Println("\t--no-parallel (default parallel):\t\t\tprevent tests from running in parallel") fmt.Println("\t--user-agent (default okapi UA):\t\t\tset the default user agent") - fmt.Println("\t--content-type (default application/json):\t\tset the default content type for requests") - fmt.Println("\t--accept (default application/json):\t\t\tset the default accept header for responses") + fmt.Println("\t--content-type (default 'application/json'):\t\tset the default content type for requests") + fmt.Println("\t--accept (default 'application/json'):\t\t\tset the default accept header for responses") fmt.Println() fmt.Println("The parameters are:") fmt.Println() From f9a403ce706af0135ed1d350c8f4388b0f59bbc3 Mon Sep 17 00:00:00 2001 From: Fred Date: Thu, 3 Aug 2023 15:43:53 +0200 Subject: [PATCH 11/12] Add link to full documentation in help command --- main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.go b/main.go index f680318..68a0265 100644 --- a/main.go +++ b/main.go @@ -33,6 +33,8 @@ func help() { fmt.Println() fmt.Println("\ttest_directory:\t\t\t\t\t\tpoint to the directory where all the test files are located") fmt.Println() + fmt.Println("More information (and source code) on: https://github.com/fred1268/okapi") + fmt.Println() } func main() { From 44e1746354eb15ab6b35bdb2bf1dc9bd705d1620 Mon Sep 17 00:00:00 2001 From: Fred Date: Thu, 3 Aug 2023 15:49:50 +0200 Subject: [PATCH 12/12] Document --workers --- README.md | 2 ++ main.go | 1 + 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index 17b01e6..738b0c5 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,8 @@ where options are one or more of the following: - `--no-parallel` (default parallel): prevent tests from running in parallel +- `--workers` (default #cores): define the maximum number of workers + - `--user-agent` (default okapi UA): set the default user agent - `--content-type` (default application/json): set the default content type for requests diff --git a/main.go b/main.go index 68a0265..75e4b13 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,7 @@ func help() { fmt.Println("\t--test, -t (default none):\t\t\t\tonly run the specified standalone test") fmt.Println("\t--timeout (default 30s):\t\t\t\tset a default timeout for all HTTP requests") fmt.Println("\t--no-parallel (default parallel):\t\t\tprevent tests from running in parallel") + fmt.Println("\t--workers (default #cores):\t\t\t\tdefine the maximum number of workers") fmt.Println("\t--user-agent (default okapi UA):\t\t\tset the default user agent") fmt.Println("\t--content-type (default 'application/json'):\t\tset the default content type for requests") fmt.Println("\t--accept (default 'application/json'):\t\t\tset the default accept header for responses")