From 3ec61a1c46085e09f2b99289da050162bbbe0efa Mon Sep 17 00:00:00 2001 From: Fred Date: Thu, 13 Jul 2023 10:12:38 +0200 Subject: [PATCH] First release --- .gitignore | 2 + README.md | 191 ++++++++++++++++++++- assets/config.json | 5 + assets/tests/121005.expected.json | 11 ++ assets/tests/hackernews.items.test.json | 114 +++++++++++++ assets/tests/hackernews.users.test.json | 49 ++++++ go.mod | 5 + go.sum | 2 + main.go | 19 +++ testing/api.go | 18 ++ testing/client.go | 183 +++++++++++++++++++++ testing/config.go | 28 ++++ testing/errors.go | 9 + testing/internal/json/json.go | 56 +++++++ testing/internal/json/json_test.go | 65 ++++++++ testing/internal/log/log.go | 12 ++ testing/server.go | 29 ++++ testing/test.go | 210 ++++++++++++++++++++++++ 18 files changed, 1007 insertions(+), 1 deletion(-) create mode 100644 assets/config.json create mode 100644 assets/tests/121005.expected.json create mode 100644 assets/tests/hackernews.items.test.json create mode 100644 assets/tests/hackernews.users.test.json create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 testing/api.go create mode 100644 testing/client.go create mode 100644 testing/config.go create mode 100644 testing/errors.go create mode 100644 testing/internal/json/json.go create mode 100644 testing/internal/json/json_test.go create mode 100644 testing/internal/log/log.go create mode 100644 testing/server.go create mode 100644 testing/test.go diff --git a/.gitignore b/.gitignore index 3b735ec..cae6d25 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ # Go workspace file go.work +.devcontainer +.vscode diff --git a/README.md b/README.md index 0947aee..3edd038 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,191 @@ -# okapi +# okapi :giraffe: + API tests made as easy as table driven tests. + +## Introduction + +okapi is a program allowing you to test your APIs by using tests files and test cases, pretty much like the `go test` command with table-driven tests. okapi will iterate on all `.test.json` files in the specified directory and runs every test case containted within the files, sequentially or in parallel. + +The result of each test case is then compared to the expected result (both the HTTP Response Status Code, as well as the payload). Success or failure are reported. + +## Setup + +Pretty easy to install: + +```shell +go install github.com/fred1268/okapi@latest +``` + +> Also, note that okapi uses [**clap :clap:, the Command Line Argument Parser**](https://github.com/fred1268/go-clap). clap is a non intrusive, lightweight command line parsing library you may want to try out in your own projects. Feel free to give it a try! + +## Configuring okapi :giraffe: + +In order to run, okapi requires at least two files: + +- a configuration file +- one or more test files +- zero or more result files + +### Configuration file + +The configuration file looks like the following: + +```json +{ + "server1": { + "host": "http://localhost:8080", + "auth": { + "login": { + "method": "POST", + "endpoint": "/login", + "payload": "{\"email\":\"test@test.com\",\"password\":\"my_password\"}", + "expected": { + "statuscode": 200 + } + } + }, + "session": { + "cookie": { + "name": "jsessionid" + } + } + }, + "server2": { + "host": "http://localhost:9090", + "auth": { + "login": { + "method": "POST", + "endpoint": "http://localhost:8080/login", + "payload": "{\"email\":\"test@test.com\",\"password\":\"my_password\"}", + "expected": { + "statuscode": 200 + } + } + }, + "session": { + "jwt": "header" + } + }, + "server3": { + "host": "http://localhost:8088", + "auth": { + "apikey": { + "apikey": "Bearer: my_key", + "header": "Authorization" + } + } + }, + "hackernews": { + "host": "https://hacker-news.firebaseio.com" + } +} +``` + +A Server description contains various fields: + +- `host`: the URL of the server (including port and everything you don't want to repeat on every test) +- `auth.login`: the information required to login the user, using the same format as a test (see below) +- `session.cookie`: name of the cookie maintaining the session + +Here `server1` uses the `/login` endpoint on the same HTTP server than the one used for the tests. Both `email` and `password` are submitted in the `POST`, and `200 OK` is expected upon successful login. The session is maintained by a session cookie called `jsessionid`. + +The second server, `server2` also uses the `/login` endpoint, but on a different server, hence the fully qualified URL given as the endpoint. The sesssion is maintained using a JWT (JSON Web Token) which is obtained though a header (namely `Authorization`). Should your JWT be returned as a payload, you can specify `"payload"` instead of `"header"`. JWT is always sent back using the `Authorization` header in the form of `Authorization: Bearer my_jwt`. + +The third server, `server3` uses API Keys for authentication. The apikey field contains the key itself, whereas the header field contains the field used to send the API Key back (usually `Authorization`). Please note that session is not maintained in this example, since the API Key is sent with each request. + +The last server, `hackernews`, is a server which doesn't require any authentication. + +### Test files + +A test file looks like the following: + +```json +{ + "tests": [ + { + "name": "121003", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121003.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "121004", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121004.json", + "expected": { + "statuscode": 200, + "result": "{\"id\":121004}" + } + }, + { + "name": "121005", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121005.json", + "expected": { + "statuscode": 200, + "result": "@file" + } + } + ] +} +``` + +> The test files must end with `.test.json` in order for okapi to find them. A good pratice is to name them based on your routes. For example, in this case, since we are testing hackernews' `item` route, the file could be named `item.test.json` or `item.get.test.json` if you need to be more specific. + +A test file contains an array of tests, each of them containing: + +- `name`: a unique name to globally identify the test +- `server`: the name of the server used to perform the test (declared in the configuration file) +- `method`: the method to perform the operation (`GET`, `POST`, `PUT`, etc.) +- `endpoint`: the endpoint of the operation (usually a ReST API of some sort) +- `expected`: this section contains: + - `statuscode`: the expected status code returned by the endpoint (200, 401, 403, etc.) + - `result`: the expected payload returned by the endpoint. This field is optional. + +> Please note that result can be either a string (including json, like shown on 121004), or a `@file` (like in 121005) if you prefer to separate the test from its expected result. This can be handy if the result are complex JSON structs that you can easily copy and paste from somewhere else, or simply want to avoid escaping double quotes. + +### Result file + +A result file doesn't have a specific format, since it represents whatever the server you are testing returns. The only important thing about the result file, is that it must be named `.expected.json` within the current directory, where `name` is the name of the current test. + +## Expected result + +As we saw earlier, for each test, you will have to define the expected result. okapi will always compare the HTTP Response Status Code with the one provided, and can optionally, compare the returned payload. The way it works is pretty simple: + +- if the result is in JSON format: + - if a field is present in expected, okapi will also check for its presence in the result + - if the result contains other fields not mentioned in `expected`, they will be ignored +- if the result is a non-JSON string: + - the result is compared to `expected` and success or failure is reported + +## Running okapi :giraffe: + +To launch okapi, please run the following: + +```shell + okapi test_directory +``` + +where options are one or more of the following: + +- `test_directory`: the directory where all the test files are located +- `--servers-file` or `-s` (mandatory): allows to point to the configuration file +- `--timeout` (default 30s): allows to set a default timeout for all HTTP requests +- `--verbose` (`--no-verbose`) or `-v` (default no): verbose mode +- `--parallel` (`--no-parallel`) or `-p` (default yes): run tests in parallel +- `--user-agent` (default okapi ua): to set the default user agent +- `--content-type` (default application/json): to set the default content type for requests +- `--accept` (default application/json): to set the default accept header for responses + +## Integration okapi :giraffe: with your own software + +okapi exposes a pretty simple and straightforward API that you can use within your own Go programs. + +## Feedback + +Feel free to send feedback, star, issues, etc. diff --git a/assets/config.json b/assets/config.json new file mode 100644 index 0000000..fc85806 --- /dev/null +++ b/assets/config.json @@ -0,0 +1,5 @@ +{ + "hackernews": { + "host": "https://hacker-news.firebaseio.com" + } +} \ No newline at end of file diff --git a/assets/tests/121005.expected.json b/assets/tests/121005.expected.json new file mode 100644 index 0000000..b8b86cd --- /dev/null +++ b/assets/tests/121005.expected.json @@ -0,0 +1,11 @@ +{ + "by": "paulgb", + "id": 121005, + "kids": [ + 121021 + ], + "parent": 120890, + "text": "I wish I had Mario. Instead I was stuck with Mavis Beacon.", + "time": 1203647923, + "type": "comment" +} \ No newline at end of file diff --git a/assets/tests/hackernews.items.test.json b/assets/tests/hackernews.items.test.json new file mode 100644 index 0000000..6bbded9 --- /dev/null +++ b/assets/tests/hackernews.items.test.json @@ -0,0 +1,114 @@ +{ + "tests": [ + { + "name": "121003", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121003.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "121004", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121004.json", + "expected": { + "statuscode": 200, + "result": "{\"id\":121004}" + } + }, + { + "name": "121005", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121005.json", + "expected": { + "statuscode": 200, + "result": "@file" + } + }, + { + "name": "121006", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121006.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "121007", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121007.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "121008", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121008.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "121009", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121009.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "121010", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121010.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "121011", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121011.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "121012", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121012.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "121013", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121013.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "121014", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/item/121014.json", + "expected": { + "statuscode": 200 + } + } + ] +} \ No newline at end of file diff --git a/assets/tests/hackernews.users.test.json b/assets/tests/hackernews.users.test.json new file mode 100644 index 0000000..b2f72aa --- /dev/null +++ b/assets/tests/hackernews.users.test.json @@ -0,0 +1,49 @@ +{ + "tests": [ + { + "name": "jk", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/user/jk.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "jl", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/user/jl.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "jc", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/user/jc.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "jo", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/user/jo.json", + "expected": { + "statuscode": 200 + } + }, + { + "name": "401", + "server": "hackernews", + "method": "GET", + "endpoint": "/v0/users/jl.json", + "expected": { + "statuscode": 401 + } + } + ] +} \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ac03cba --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/fred1268/okapi + +go 1.20 + +require github.com/fred1268/go-clap v1.1.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5aea60c --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/fred1268/go-clap v1.1.0 h1:bxgST/3OvEls2mzkZaxbX5322x7bnSL5VnxAnYQNxIs= +github.com/fred1268/go-clap v1.1.0/go.mod h1:A5/yYBapOy6UyujlbxL7p/bX9J7bzyoMRzQKFwveXF0= diff --git a/main.go b/main.go new file mode 100644 index 0000000..cffd464 --- /dev/null +++ b/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "context" + "log" + "os" + + "github.com/fred1268/okapi/testing" +) + +func main() { + cfg, err := testing.LoadConfig(os.Args) + if err != nil { + log.Fatalf("Cannot read command line parameters: %s\n", err) + } + if err := testing.Run(context.Background(), cfg); err != nil { + log.Fatalf("Cannot run tests: %s\n", err) + } +} diff --git a/testing/api.go b/testing/api.go new file mode 100644 index 0000000..b78e54e --- /dev/null +++ b/testing/api.go @@ -0,0 +1,18 @@ +package testing + +type APIRequest struct { + Name string + Server string + Method string + Endpoint string + Headers map[string]string + URLParams map[string]string + Payload string + Expected *APIResponse +} + +type APIResponse struct { + StatusCode int + Response string + Logs []string +} diff --git a/testing/client.go b/testing/client.go new file mode 100644 index 0000000..f233383 --- /dev/null +++ b/testing/client.go @@ -0,0 +1,183 @@ +package testing + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/fred1268/okapi/testing/internal/json" +) + +type Client struct { + config *ServerConfig + client *http.Client + cookie *http.Cookie + jwt string +} + +func NewClient(config *ServerConfig) *Client { + client := &Client{ + config: config, + client: &http.Client{ + Timeout: time.Duration(config.Timeout) * time.Second, + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 5 * time.Second, + }).DialContext, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 10 * time.Second, + MaxIdleConns: 100, + MaxConnsPerHost: 100, + MaxIdleConnsPerHost: 100, + }, + }, + } + return client +} + +func (c *Client) buildEndpointURL(ctx context.Context, apiRequest *APIRequest) (string, error) { + var err error + addr := apiRequest.Endpoint + if !strings.Contains(apiRequest.Endpoint, "://") { + addr, err = url.JoinPath(c.config.Host, apiRequest.Endpoint) + if err != nil { + return "", err + } + } + if len(apiRequest.URLParams) != 0 { + first := true + for key, value := range apiRequest.URLParams { + if first { + addr = fmt.Sprintf("%s?%s=%s", addr, url.QueryEscape(key), url.QueryEscape(value)) + first = false + } else { + addr = fmt.Sprintf("%s&%s=%s", addr, url.QueryEscape(key), url.QueryEscape(value)) + } + } + } + return addr, nil +} + +func (c *Client) getRequest(ctx context.Context, apiRequest *APIRequest) (*http.Request, error) { + addr, err := c.buildEndpointURL(ctx, apiRequest) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, strings.ToUpper(apiRequest.Method), addr, + bytes.NewBufferString(apiRequest.Payload)) + if err != nil { + return nil, err + } + if c.config.Auth != nil && c.config.Auth.APIKey != nil { + req.Header.Set(c.config.Auth.APIKey.Header, c.config.Auth.APIKey.APIKey) + } + req.Header.Add("User-Agent", c.config.UserAgent) + if c.jwt != "" { + req.Header.Set("authorization", fmt.Sprintf("Bearer: %s", c.jwt)) + } + if c.cookie != nil { + req.AddCookie(c.cookie) + } + if len(apiRequest.Headers) != 0 { + for key, value := range apiRequest.Headers { + req.Header.Add(key, value) + } + } else { + for key, value := range c.config.Headers { + req.Header.Add(key, value) + } + } + return req, nil +} + +func (c *Client) call(ctx context.Context, apiRequest *APIRequest) (response *APIResponse, err error) { + var req *http.Request + req, err = c.getRequest(ctx, apiRequest) + if err != nil { + return + } + var resp *http.Response + resp, err = c.client.Do(req) + if err != nil { + return + } + defer func() { + err = resp.Body.Close() + }() + if c.cookie == nil && c.config.Session != nil && c.config.Session.Cookie.Name != "" { + cookies := resp.Cookies() + for _, cookie := range cookies { + if cookie.Name == c.config.Session.Cookie.Name { + c.cookie = cookie + break + } + } + } + var res []byte + res, err = io.ReadAll(resp.Body) + if err != nil { + return + } + if c.jwt == "" && c.config.Session != nil { + switch c.config.Session.JWT { + case "payload": + // TODO use something like {token:"..."} + c.jwt = string(res) + case "header": + c.jwt = resp.Header.Get("authorization") + default: + err = ErrInvalidJWTSession + } + } + response = &APIResponse{StatusCode: resp.StatusCode, Response: string(res)} + return +} + +func (c *Client) Connect(ctx context.Context) (*APIResponse, error) { + result, err := c.call(ctx, c.config.Auth.Login) + if err != nil { + return nil, err + } + if result.StatusCode != c.config.Auth.Login.Expected.StatusCode { + return result, ErrStatusCodeMismatched + } + return result, nil +} + +func (c *Client) Test(ctx context.Context, apiRequest *APIRequest, verbose bool) (response *APIResponse, err error) { + start := time.Now() + defer func() { + if err == nil { + if verbose { + response.Logs = append(response.Logs, fmt.Sprintf(" --- PASS:\t%s (%0.2fs)\n", apiRequest.Name, + time.Since(start).Seconds())) + } + } else { + response.Logs = append(response.Logs, fmt.Sprintf(" --- FAIL:\t%s (%0.2fs)\n", apiRequest.Name, + time.Since(start).Seconds())) + response.Logs = append(response.Logs, fmt.Sprintf(" wanted: '%s' (%d), got '%s' (%d)\n", + apiRequest.Expected.Response, apiRequest.Expected.StatusCode, strings.Trim(response.Response, "\n"), + response.StatusCode)) + } + }() + response, err = c.call(ctx, apiRequest) + if err != nil { + return + } + if response.StatusCode != apiRequest.Expected.StatusCode { + err = ErrStatusCodeMismatched + return + } + err = json.CompareJSONStrings(ctx, apiRequest.Expected.Response, response.Response) + if errors.Is(err, json.ErrJSONMismatched) { + err = errors.Join(err, ErrResultMismatched) + } + return +} diff --git a/testing/config.go b/testing/config.go new file mode 100644 index 0000000..4054448 --- /dev/null +++ b/testing/config.go @@ -0,0 +1,28 @@ +package testing + +import "github.com/fred1268/go-clap/clap" + +type Config struct { + Servers string `clap:"--servers-file,-s,mandatory"` + Tests string `clap:"trailing"` + Timeout int `clap:"--timeout"` + UserAgent string `clap:"--user-agent"` + ContentType string `clap:"--content-type"` + Accept string `clap:"--accept"` + Verbose bool `clap:"--verbose,-v"` + Parallel bool `clap:"--parallel,-p"` +} + +func LoadConfig(args []string) (*Config, error) { + var cfg Config = Config{ + Timeout: 30, + UserAgent: "Mozilla/5.0 (compatible; okapi/1.0; +https://github.com/fred1268/okapi)", + ContentType: "application/json", + Accept: "application/json", + Parallel: true, + } + if _, err := clap.Parse(args, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} diff --git a/testing/errors.go b/testing/errors.go new file mode 100644 index 0000000..96ae189 --- /dev/null +++ b/testing/errors.go @@ -0,0 +1,9 @@ +package testing + +import "errors" + +var ( + ErrStatusCodeMismatched error = errors.New("status code mismatched") + ErrResultMismatched error = errors.New("result mismatched") + ErrInvalidJWTSession error = errors.New("invalid JWT session") +) diff --git a/testing/internal/json/json.go b/testing/internal/json/json.go new file mode 100644 index 0000000..8d58479 --- /dev/null +++ b/testing/internal/json/json.go @@ -0,0 +1,56 @@ +package json + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" +) + +var ErrJSONMismatched error = errors.New("json mismatched") + +func compareMaps(ctx context.Context, src, dst map[string]any) error { + for key, value := range src { + dstValue, found := dst[key] + if !found { + return ErrJSONMismatched + } + if reflect.TypeOf(value) != reflect.TypeOf(dstValue) { + return ErrJSONMismatched + } + switch value.(type) { + case nil: + case map[string]any: + if err := compareMaps(ctx, value.(map[string]any), dstValue.(map[string]any)); err != nil { + return err + } + default: + if value != dstValue { + return ErrJSONMismatched + } + } + } + return nil +} + +func CompareJSONStrings(ctx context.Context, wanted, got string) error { + if wanted == "" || got == wanted { + return nil + } + var g, w interface{} + err := json.Unmarshal([]byte(strings.ToLower(got)), &g) + if err != nil { + return err + } + err = json.Unmarshal([]byte(strings.ToLower(wanted)), &w) + if err != nil { + return err + } + if wantedMap, ok := w.(map[string]any); ok { + if gotMap, ok := g.(map[string]any); ok { + return compareMaps(ctx, wantedMap, gotMap) + } + } + return ErrJSONMismatched +} diff --git a/testing/internal/json/json_test.go b/testing/internal/json/json_test.go new file mode 100644 index 0000000..627de3b --- /dev/null +++ b/testing/internal/json/json_test.go @@ -0,0 +1,65 @@ +package json + +import ( + "context" + "testing" +) + +func TestCompareJSON(t *testing.T) { + tests := []struct { + name string + src string + dst string + result error + }{ + { + name: "empty", + src: "{}", + dst: "{}", + result: nil, + }, + { + name: "identical", + src: "{\"id\":\"10\",\"field1\":\"value1\"}", + dst: "{\"id\":\"10\",\"field1\":\"value1\"}", + result: nil, + }, + { + name: "larger", + src: "{\"id\":\"10\",\"field1\":\"value1\"}", + dst: "{\"id\":\"10\",\"field1\":\"value1\",\"field2\":\"value2\"}", + result: nil, + }, + { + name: "smaller", + src: "{\"id\":\"10\",\"field1\":\"value1\",\"field2\":\"value2\",\"field3\":\"value3\"}", + dst: "{\"id\":\"10\",\"field1\":\"value1\",\"field2\":\"value2\"}", + result: ErrJSONMismatched, + }, + { + name: "embedded but identical", + src: "{\"id\":\"10\",\"field1\":\"value1\",\"field2\":{\"field21\":\"value2\"}}", + dst: "{\"id\":\"10\",\"field1\":\"value1\",\"field2\":{\"field21\":\"value2\"}}", + result: nil, + }, + { + name: "embedded but different", + src: "{\"id\":10,\"field1\":\"value1\",\"field2\":{\"field21\":\"value2\"}}", + dst: "{\"id\":10,\"field1\":\"value1\",\"field2\":{\"field21\":\"value3\"}}", + result: ErrJSONMismatched, + }, + } + for _, tt := range tests { + name := tt.name + src := tt.src + dst := tt.dst + res := tt.result + t.Run(name, func(t *testing.T) { + t.Parallel() + err := CompareJSONStrings(context.Background(), src, dst) + if err != res { + t.Errorf("wanted: '%s', got '%s'", dst, src) + } + }) + } +} diff --git a/testing/internal/log/log.go b/testing/internal/log/log.go new file mode 100644 index 0000000..0cf8d77 --- /dev/null +++ b/testing/internal/log/log.go @@ -0,0 +1,12 @@ +package log + +import "fmt" + +func Printf(format string, args ...any) { + fmt.Printf(format, args...) +} + +func Fatalf(format string, args ...any) { + format = fmt.Sprintf("FAIL\t%s", format) + fmt.Printf(format, args...) +} diff --git a/testing/server.go b/testing/server.go new file mode 100644 index 0000000..652e9a2 --- /dev/null +++ b/testing/server.go @@ -0,0 +1,29 @@ +package testing + +type AuthenticationAPIKey struct { + APIKey string + Header string +} + +type Authentication struct { + Login *APIRequest + APIKey *AuthenticationAPIKey +} + +type SessionCookie struct { + Name string +} + +type Session struct { + Cookie *SessionCookie + JWT string +} + +type ServerConfig struct { + Host string + Headers map[string]string + Auth *Authentication + Session *Session + UserAgent string + Timeout int +} diff --git a/testing/test.go b/testing/test.go new file mode 100644 index 0000000..98fdb5c --- /dev/null +++ b/testing/test.go @@ -0,0 +1,210 @@ +package testing + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path" + "runtime" + "strings" + "sync" + "time" + + "github.com/fred1268/okapi/testing/internal/log" +) + +type testIn struct { + file string + test *APIRequest + client *Client + start time.Time + verbose bool +} + +type testOut struct { + file string + start time.Time + fail bool + logs []string +} + +func readServersConfigs(filename string) (map[string]*ServerConfig, error) { + if _, err := os.Stat(filename); os.IsNotExist(err) { + return nil, fmt.Errorf("file '%s' does not exist: %w", filename, err) + } + content, err := os.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("cannot open file '%s': %w", filename, err) + } + var config map[string]*ServerConfig + if err = json.NewDecoder(bytes.NewReader(content)).Decode(&config); err != nil { + return nil, fmt.Errorf("cannot decode file '%s': %w", filename, err) + } + return config, nil +} + +func readExpectedJSON(directory string, requests []*APIRequest) error { + for _, request := range requests { + if request.Expected.Response != "@file" { + continue + } + file := fmt.Sprintf("%s.expected.json", strings.ToLower(request.Name)) + content, err := os.ReadFile(path.Join(directory, file)) + if err != nil { + return fmt.Errorf("cannot read test file '%s': %w", file, err) + } + request.Expected.Response = string(content) + } + return nil +} + +func readTestFiles(directory string) (map[string][]*APIRequest, error) { + files, err := os.ReadDir(directory) + if err != nil { + return nil, err + } + allTests := make(map[string][]*APIRequest) + for _, file := range files { + if !strings.HasSuffix(file.Name(), ".test.json") { + continue + } + content, err := os.ReadFile(path.Join(directory, file.Name())) + if err != nil { + return nil, fmt.Errorf("cannot read test file '%s': %w", file.Name(), err) + } + var tests struct { + Tests []*APIRequest + } + if err = json.NewDecoder(bytes.NewReader(content)).Decode(&tests); err != nil { + return nil, fmt.Errorf("cannot decode json file '%s': %w", file.Name(), err) + } + readExpectedJSON(directory, tests.Tests) + allTests[file.Name()] = tests.Tests + } + return allTests, nil +} + +func connect(ctx context.Context, cfg *Config) (map[string]*Client, error) { + serverConfigs, err := readServersConfigs(cfg.Servers) + if err != nil { + return nil, err + } + clients := make(map[string]*Client) + for key, value := range serverConfigs { + client := NewClient(value) + if client.config.Auth != nil { + if apiResponse, err := client.Connect(ctx); err != nil { + return nil, fmt.Errorf("cannot connect to server '%s': %w (%v)", key, err, apiResponse) + } + if cfg.Verbose { + log.Printf("Connected to %s (%s)\n", key, value.Host) + } + } + clients[key] = client + } + return clients, nil +} + +func runOne(ctx context.Context, tin *testIn, out chan<- *testOut) error { + tout := &testOut{file: tin.file, start: tin.start} + response, err := tin.client.Test(ctx, tin.test, tin.verbose) + if err != nil { + if !errors.Is(err, ErrStatusCodeMismatched) && !errors.Is(err, ErrResultMismatched) { + return fmt.Errorf("cannot run test '%s': %w", tin.file, err) + } + tout.fail = true + } + tout.logs = append(tout.logs, response.Logs...) + out <- tout + return nil +} + +func Run(ctx context.Context, cfg *Config) error { + clients, err := connect(ctx, cfg) + if err != nil { + return fmt.Errorf("cannot connect to servers: %w", err) + } + allTests, err := readTestFiles(cfg.Tests) + if err != nil { + return fmt.Errorf("cannot read tests: %w", err) + } + out := make(chan *testOut) + in := make(chan *testIn) + if cfg.Parallel { + in = make(chan *testIn, runtime.NumCPU()) + } + startTimes := make(map[string]time.Time) + results := make(map[string]struct{}) + var wg sync.WaitGroup + go func() { + for { + run := <-in + if err := runOne(ctx, run, out); err != nil { + log.Fatalf("runOne failed: %v\n", err) + wg.Done() + return + } + } + }() + go func() { + counts := make(map[string]int) + logs := make(map[string][]string) + for { + tout := <-out + counts[tout.file]++ + if tout.fail { + results[tout.file] = struct{}{} + } + logs[tout.file] = append(logs[tout.file], tout.logs...) + if counts[tout.file] == len(allTests[tout.file]) { + lines := logs[tout.file] + delete(logs, tout.file) + delete(counts, tout.file) + if _, ok := results[tout.file]; ok { + log.Printf("--- FAIL:\t%s\n", tout.file) + } else if cfg.Verbose { + log.Printf("--- PASS:\t%s\n", tout.file) + } + for _, line := range lines { + log.Printf(line) + } + if _, ok := results[tout.file]; !ok && cfg.Verbose { + log.Printf("PASS\n") + } + if _, ok := results[tout.file]; ok { + log.Printf("FAIL \n") + log.Printf("FAIL\t%s\t\t\t%0.3fs\n", tout.file, time.Since(startTimes[tout.file]).Seconds()) + log.Printf("FAIL \n") + } else { + log.Printf("ok\t%-30s\t\t\t%0.3fs\n", tout.file, time.Since(startTimes[tout.file]).Seconds()) + } + } + wg.Done() + } + }() + for key, tests := range allTests { + startTimes[key] = time.Now() + for _, test := range tests { + if clients[test.Server] == nil { + log.Fatalf("invalid server for %s ('%s')\n", test.Name, key) + continue + } + tin := &testIn{ + file: key, + test: test, + client: clients[test.Server], + start: time.Now(), + verbose: cfg.Verbose, + } + wg.Add(1) + in <- tin + } + } + wg.Wait() + close(in) + close(out) + return nil +}