From 8b251813e5a8bebe8bb81e999e8535abcc003a1f Mon Sep 17 00:00:00 2001 From: Mikhail Gerasimchuk Date: Mon, 8 May 2023 16:52:59 +0600 Subject: [PATCH] Add request URL transformation feature --- README.md | 5 ++-- internal/adapter/cli/start.go | 1 + .../config/start_command_config.go | 1 + .../infrastructure/service/reverse_proxy.go | 30 ++++++++++++++++--- pkg/util/jq.go | 2 +- pkg/util/sed.go | 2 +- 6 files changed, 33 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 1aeb13d..be43782 100644 --- a/README.md +++ b/README.md @@ -18,13 +18,13 @@ These capabilities make Protty a useful tool for a variety of purposes, such as The following command will start a proxy on port 8080, and after starting, all traffic from port 8080 will be redirected to a remote host located at https://example.com ```shell -docker run -p8080:80 -e REMOTE_URI=https://example.com:443 mgerasimchuk/protty:v0.3.0 +docker run -p8080:80 -e REMOTE_URI=https://example.com:443 mgerasimchuk/protty:v0.4.0 ``` ## Running options and runtime configuration ``` -» ~ docker run -p8080:80 -it mgerasimchuk/protty:v0.3.0 /bin/sh -c 'protty start --help' +» ~ docker run -p8080:80 -it mgerasimchuk/protty:v0.4.0 /bin/sh -c 'protty start --help' Start the proxy Usage: @@ -63,6 +63,7 @@ Flags: --local-port int Verbosity level (panic, fatal, error, warn, info, debug, trace) | Env variable alias: LOCAL_PORT | Request header alias: X-PROTTY-LOCAL-PORT (default 80) --remote-uri string Listening port for the proxy | Env variable alias: REMOTE_URI | Request header alias: X-PROTTY-REMOTE-URI (default "https://example.com:443") --throttle-rate-limit float How many requests can be send to the remote resource per second | Env variable alias: THROTTLE_RATE_LIMIT | Request header alias: X-PROTTY-THROTTLE-RATE-LIMIT + --transform-request-url-sed string SED expression for request URL transformation | Env variable alias: TRANSFORM_REQUEST_URL_SED | Request header alias: X-PROTTY-TRANSFORM-REQUEST-URL-SED --additional-request-headers stringArray Array of additional request headers in format Header: Value | Env variable alias: ADDITIONAL_REQUEST_HEADERS | Request header alias: X-PROTTY-ADDITIONAL-REQUEST-HEADERS --transform-request-body-sed stringArray Pipeline of SED expressions for request body transformation | Env variable alias: TRANSFORM_REQUEST_BODY_SED | Request header alias: X-PROTTY-TRANSFORM-REQUEST-BODY-SED --transform-request-body-jq stringArray Pipeline of JQ expressions for request body transformation | Env variable alias: TRANSFORM_REQUEST_BODY_JQ | Request header alias: X-PROTTY-TRANSFORM-REQUEST-BODY-JQ diff --git a/internal/adapter/cli/start.go b/internal/adapter/cli/start.go index a640a20..c32337d 100644 --- a/internal/adapter/cli/start.go +++ b/internal/adapter/cli/start.go @@ -36,6 +36,7 @@ func NewStartCommand(cfg *config.StartCommandConfig, reverseProxySvc *service.Re startCommand.cobraCmd.Flags().IntVar(buildFlagArgs(&cfg.LocalPort)) startCommand.cobraCmd.Flags().StringVar(buildFlagArgs(&cfg.RemoteURI)) startCommand.cobraCmd.Flags().Float64Var(buildFlagArgs(&cfg.ThrottleRateLimit)) + startCommand.cobraCmd.Flags().StringVar(buildFlagArgs(&cfg.TransformRequestUrlSED)) startCommand.cobraCmd.Flags().StringArrayVar(buildFlagArgs(&cfg.AdditionalRequestHeaders)) startCommand.cobraCmd.Flags().StringArrayVar(buildFlagArgs(&cfg.TransformRequestBodySED)) startCommand.cobraCmd.Flags().StringArrayVar(buildFlagArgs(&cfg.TransformRequestBodyJQ)) diff --git a/internal/infrastructure/config/start_command_config.go b/internal/infrastructure/config/start_command_config.go index 37a7e06..0fabda0 100644 --- a/internal/infrastructure/config/start_command_config.go +++ b/internal/infrastructure/config/start_command_config.go @@ -19,6 +19,7 @@ type StartCommandConfig struct { LocalPort Option[int] `default:"80" description:"Verbosity level (panic, fatal, error, warn, info, debug, trace)"` RemoteURI Option[string] `default:"https://example.com:443" description:"Listening port for the proxy"` ThrottleRateLimit Option[float64] `description:"How many requests can be send to the remote resource per second"` + TransformRequestUrlSED Option[string] `description:"SED expression for request URL transformation"` AdditionalRequestHeaders Option[[]string] `description:"Array of additional request headers in format Header: Value"` TransformRequestBodySED Option[[]string] `description:"Pipeline of SED expressions for request body transformation"` TransformRequestBodyJQ Option[[]string] `description:"Pipeline of JQ expressions for request body transformation"` diff --git a/internal/infrastructure/service/reverse_proxy.go b/internal/infrastructure/service/reverse_proxy.go index d4c4616..ba3a6ca 100644 --- a/internal/infrastructure/service/reverse_proxy.go +++ b/internal/infrastructure/service/reverse_proxy.go @@ -38,12 +38,12 @@ func (s *ReverseProxyService) Start(cfg *config.StartCommandConfig) error { } func (s *ReverseProxyService) handleRequestAndRedirect(res http.ResponseWriter, req *http.Request) { - s.logRequestPayload(req) s.serveReverseProxy(res, req) + s.logRequestPayload(req) } func (s *ReverseProxyService) logRequestPayload(req *http.Request) { - s.logger.WithField("method", req.Method).WithField("path", req.URL.Path).Infof("Send request to %s", req.URL.Host) + s.logger.WithField("method", req.Method).WithField("path", req.URL.Path).Infof("Request have been sent to %s", req.URL.Host) // TODO add tracing log with body and other params like headers } @@ -61,6 +61,22 @@ func (s *ReverseProxyService) modifyRequest(cfg config.StartCommandConfig, req * // Deleting encoding to keep availability for changing response req.Header.Del("Accept-Encoding") + // Transform request URL + if cfg.TransformRequestUrlSED.Value != "" { + if modifiedURLRaw, _, err := util.SED(cfg.TransformRequestUrlSED.Value, []byte(req.URL.String())); err == nil { + sourceURLRaw := req.URL.String() + if modifiedURL, err := url.Parse(strings.Trim(string(modifiedURLRaw), "\n")); err == nil { // TODO remove trim (currently it is a hotfix, cos the util.SED added \n at the end unexpectedly) + req.URL = modifiedURL + s.logger.Debugf("ModifyRequestURL: %s", getChangesLogMessage([]byte(sourceURLRaw), modifiedURLRaw, cfg.TransformRequestUrlSED.Value, cfg.TransformRequestUrlSED)) + } else { + s.logger.Errorf("%s: %s: %s", util.GetCurrentFuncName(), util.GetFuncName(url.Parse), err) + return + } + } else { + s.logger.Errorf("%s: %s: %s", util.GetCurrentFuncName(), util.GetFuncName(util.SED), err) + } + } + // Add request headers for _, header := range cfg.AdditionalRequestHeaders.Value { kv := strings.SplitN(header, ": ", 2) @@ -83,6 +99,7 @@ func (s *ReverseProxyService) modifyRequest(cfg config.StartCommandConfig, req * modifiedRequestBody := sourceRequestBody + // Transform request body with SED for _, sedExpr := range cfg.TransformRequestBodySED.Value { modifiedRequestBody, sourceRequestBody, err = util.SED(sedExpr, modifiedRequestBody) if err != nil { @@ -91,6 +108,8 @@ func (s *ReverseProxyService) modifyRequest(cfg config.StartCommandConfig, req * } s.logger.Debugf("ModifyRequestBody: %s", getChangesLogMessage(sourceRequestBody, modifiedRequestBody, sedExpr, cfg.TransformRequestBodySED)) } + + // Transform request body with JQ for _, jqExpr := range cfg.TransformRequestBodyJQ.Value { modifiedRequestBody, sourceRequestBody, err = util.JQ(jqExpr, modifiedRequestBody) if err != nil { @@ -137,6 +156,7 @@ func (s *ReverseProxyService) getModifyResponseFunc(cfg config.StartCommandConfi modifiedResponseBody := sourceResponseBody + // Transform response body with SED for _, sedExpr := range cfg.TransformResponseBodySED.Value { modifiedResponseBody, sourceResponseBody, err = util.SED(sedExpr, modifiedResponseBody) if err != nil { @@ -145,6 +165,8 @@ func (s *ReverseProxyService) getModifyResponseFunc(cfg config.StartCommandConfi } s.logger.Debugf("ModifyResponseBody: %s", getChangesLogMessage(sourceResponseBody, modifiedResponseBody, sedExpr, cfg.TransformResponseBodySED)) } + + // Transform response body with SED for _, jqExpr := range cfg.TransformResponseBodyJQ.Value { modifiedResponseBody, sourceResponseBody, err = util.JQ(jqExpr, modifiedResponseBody) if err != nil { @@ -191,10 +213,10 @@ func (s *ReverseProxyService) getOverrideConfig(req *http.Request) *config.Start func getChangesLogMessage[T config.OptionValueType](source, modified []byte, expr string, o config.Option[T]) string { if string(source) == string(modified) { - return fmt.Sprintf("the '%s' %s expression didn't change the response", expr, o.Name) + return fmt.Sprintf("the '%s' %s expression didn't change the data", expr, o.Name) } else { // TODO add diff to log - return fmt.Sprintf("the '%s' %s expression changed the response. Length difference: %d", + return fmt.Sprintf("the '%s' %s expression changed the data. Length difference: %d", expr, o.Name, int(math.Abs(float64(len(source)-len(modified))))) } } diff --git a/pkg/util/jq.go b/pkg/util/jq.go index 15725d6..c89b0fb 100644 --- a/pkg/util/jq.go +++ b/pkg/util/jq.go @@ -9,7 +9,7 @@ import ( // JQ transform the input by jq expression // in the error case returns the original input func JQ(jqExpr string, input []byte) ([]byte, []byte, error) { - if len(input) == 0 { + if len(input) == 0 || len(jqExpr) == 0 { return input, input, nil } diff --git a/pkg/util/sed.go b/pkg/util/sed.go index c8b91e1..08afc03 100644 --- a/pkg/util/sed.go +++ b/pkg/util/sed.go @@ -9,7 +9,7 @@ import ( // SED replace the input by sed expression // in the error case returns the original input func SED(sedExpr string, input []byte) ([]byte, []byte, error) { - if len(input) == 0 { + if len(input) == 0 || len(sedExpr) == 0 { return input, input, nil } engine, err := sed.New(strings.NewReader(sedExpr))