diff --git a/.gitignore b/.gitignore index 56f1c98..bb390c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ # Visual Studio Code -.vscode \ No newline at end of file +.vscode +captive-core*/ \ No newline at end of file diff --git a/cmd/ingest.go b/cmd/ingest.go new file mode 100644 index 0000000..780f880 --- /dev/null +++ b/cmd/ingest.go @@ -0,0 +1,112 @@ +package cmd + +import ( + "go/types" + + _ "github.com/lib/pq" + "github.com/spf13/cobra" + "github.com/stellar/go/network" + "github.com/stellar/go/support/config" + "github.com/stellar/go/support/log" + "github.com/stellar/wallet-backend/cmd/utils" + "github.com/stellar/wallet-backend/internal/ingest" +) + +type ingestCmd struct{} + +func (c *ingestCmd) Command() *cobra.Command { + cfg := ingest.Configs{} + cfgOpts := config.ConfigOptions{ + { + Name: "database-url", + Usage: "Database connection URL.", + OptType: types.String, + ConfigKey: &cfg.DatabaseURL, + FlagDefault: "postgres://postgres@localhost:5432/wallet-backend?sslmode=disable", + Required: true, + }, + { + Name: "network-passphrase", + Usage: "Stellar Network Passphrase to connect.", + OptType: types.String, + ConfigKey: &cfg.NetworkPassphrase, + FlagDefault: network.TestNetworkPassphrase, + Required: true, + }, + { + Name: "log-level", + Usage: `The log level used in this project. Options: "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL", or "PANIC".`, + OptType: types.String, + FlagDefault: "TRACE", + ConfigKey: &cfg.LogLevel, + CustomSetValue: utils.SetConfigOptionLogLevel, + Required: false, + }, + { + Name: "captive-core-bin-path", + Usage: "Path to Captive Core's binary file.", + OptType: types.String, + CustomSetValue: utils.SetConfigOptionCaptiveCoreBinPath, + ConfigKey: &cfg.CaptiveCoreBinPath, + FlagDefault: "/usr/local/bin/stellar-core", + Required: true, + }, + { + Name: "captive-core-config-dir", + Usage: "Path to Captive Core's configuration files directory.", + OptType: types.String, + CustomSetValue: utils.SetConfigOptionCaptiveCoreConfigDir, + ConfigKey: &cfg.CaptiveCoreConfigDir, + FlagDefault: "./internal/ingest/config", + Required: true, + }, + { + Name: "ledger-cursor-name", + Usage: "Name of last synced ledger cursor, used to keep track of the last ledger ingested by the service. When starting up, ingestion will resume from the ledger number stored in this record. It should be an unique name per container as different containers would overwrite the cursor value of its peers when using the same cursor name.", + OptType: types.String, + ConfigKey: &cfg.LedgerCursorName, + Required: true, + }, + { + Name: "start", + Usage: "Ledger number from which ingestion should start. When not present, ingestion will resume from last synced ledger.", + OptType: types.Int, + ConfigKey: &cfg.StartLedger, + FlagDefault: 0, + Required: false, + }, + { + Name: "end", + Usage: "Ledger number up to which ingestion should run. When not present, ingestion run indefinitely (live ingestion requires it to be empty).", + OptType: types.Int, + ConfigKey: &cfg.EndLedger, + FlagDefault: 0, + Required: false, + }, + } + + cmd := &cobra.Command{ + Use: "ingest", + Short: "Run Ingestion service", + PersistentPreRun: func(_ *cobra.Command, _ []string) { + cfgOpts.Require() + if err := cfgOpts.SetValues(); err != nil { + log.Fatalf("Error setting values of config options: %s", err.Error()) + } + }, + Run: func(_ *cobra.Command, _ []string) { + c.Run(cfg) + }, + } + if err := cfgOpts.Init(cmd); err != nil { + log.Fatalf("Error initializing a config option: %s", err.Error()) + } + return cmd +} + +func (c *ingestCmd) Run(cfg ingest.Configs) { + err := ingest.Ingest(cfg) + if err != nil { + log.Fatalf("Error running Ingest: %s", err.Error()) + } +} diff --git a/cmd/root.go b/cmd/root.go index 7a10ffc..1e3967d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,11 +1,8 @@ package cmd import ( - "log" - - "github.com/sirupsen/logrus" "github.com/spf13/cobra" - supportlog "github.com/stellar/go/support/log" + "github.com/stellar/go/support/log" ) // rootCmd represents the base command when called without any subcommands @@ -30,8 +27,8 @@ func Execute() { } func init() { - logger := supportlog.New() - logger.SetLevel(logrus.TraceLevel) + log.DefaultLogger = log.New() - rootCmd.AddCommand((&serveCmd{Logger: logger}).Command()) + rootCmd.AddCommand((&serveCmd{}).Command()) + rootCmd.AddCommand((&ingestCmd{}).Command()) } diff --git a/cmd/serve.go b/cmd/serve.go index 4e2c524..c6a8bd2 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -6,19 +6,15 @@ import ( _ "github.com/lib/pq" "github.com/spf13/cobra" "github.com/stellar/go/support/config" - supportlog "github.com/stellar/go/support/log" + "github.com/stellar/go/support/log" "github.com/stellar/wallet-backend/cmd/utils" "github.com/stellar/wallet-backend/internal/serve" ) -type serveCmd struct { - Logger *supportlog.Entry -} +type serveCmd struct{} func (c *serveCmd) Command() *cobra.Command { - cfg := serve.Configs{ - Logger: c.Logger, - } + cfg := serve.Configs{} cfgOpts := config.ConfigOptions{ { Name: "port", @@ -44,6 +40,15 @@ func (c *serveCmd) Command() *cobra.Command { FlagDefault: "http://localhost:8000", Required: true, }, + { + Name: "log-level", + Usage: `The log level used in this project. Options: "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL", or "PANIC".`, + OptType: types.String, + FlagDefault: "TRACE", + ConfigKey: &cfg.LogLevel, + CustomSetValue: utils.SetConfigOptionLogLevel, + Required: false, + }, { Name: "wallet-signing-key", Usage: "The public key of the Stellar account that signs the payloads when making HTTP Request to this server.", @@ -59,7 +64,7 @@ func (c *serveCmd) Command() *cobra.Command { PersistentPreRun: func(_ *cobra.Command, _ []string) { cfgOpts.Require() if err := cfgOpts.SetValues(); err != nil { - c.Logger.Fatalf("Error setting values of config options: %s", err.Error()) + log.Fatalf("Error setting values of config options: %s", err.Error()) } }, Run: func(_ *cobra.Command, _ []string) { @@ -67,7 +72,7 @@ func (c *serveCmd) Command() *cobra.Command { }, } if err := cfgOpts.Init(cmd); err != nil { - c.Logger.Fatalf("Error initializing a config option: %s", err.Error()) + log.Fatalf("Error initializing a config option: %s", err.Error()) } return cmd } @@ -75,6 +80,6 @@ func (c *serveCmd) Command() *cobra.Command { func (c *serveCmd) Run(cfg serve.Configs) { err := serve.Serve(cfg) if err != nil { - c.Logger.Fatalf("Error running Serve: %s", err.Error()) + log.Fatalf("Error running Serve: %s", err.Error()) } } diff --git a/cmd/utils/custom_set_value.go b/cmd/utils/custom_set_value.go index f075a40..97ef80b 100644 --- a/cmd/utils/custom_set_value.go +++ b/cmd/utils/custom_set_value.go @@ -1,13 +1,45 @@ package utils import ( + "errors" "fmt" + "os" + "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/stellar/go/keypair" "github.com/stellar/go/support/config" + "github.com/stellar/go/support/log" ) +func unexpectedTypeError(key any, co *config.ConfigOption) error { + return fmt.Errorf("the expected type for the config key in %s is %T, but a %T was provided instead", co.Name, key, co.ConfigKey) +} + +func SetConfigOptionLogLevel(co *config.ConfigOption) error { + logLevelStr := viper.GetString(co.Name) + logLevel, err := logrus.ParseLevel(logLevelStr) + if err != nil { + return fmt.Errorf("couldn't parse log level in %s: %w", co.Name, err) + } + + key, ok := co.ConfigKey.(*logrus.Level) + if !ok { + return fmt.Errorf("%s configKey has an invalid type %T", co.Name, co.ConfigKey) + } + *key = logLevel + + // Log for debugging + if config.IsExplicitlySet(co) { + log.Debugf("Setting log level to: %s", logLevel) + log.DefaultLogger.SetLevel(*key) + } else { + log.Debugf("Using default log level: %s", logLevel) + } + + return nil +} + func SetConfigOptionStellarPublicKey(co *config.ConfigOption) error { publicKey := viper.GetString(co.Name) @@ -18,9 +50,51 @@ func SetConfigOptionStellarPublicKey(co *config.ConfigOption) error { key, ok := co.ConfigKey.(*string) if !ok { - return fmt.Errorf("the expected type for the config key in %s is a string, but a %T was provided instead", co.Name, co.ConfigKey) + return unexpectedTypeError(key, co) } *key = kp.Address() return nil } + +func SetConfigOptionCaptiveCoreBinPath(co *config.ConfigOption) error { + binPath := viper.GetString(co.Name) + + fileInfo, err := os.Stat(binPath) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("binary file %s does not exist", binPath) + } + + if fileInfo.IsDir() { + return fmt.Errorf("binary file path %s is a directory, not a file", binPath) + } + + key, ok := co.ConfigKey.(*string) + if !ok { + return unexpectedTypeError(key, co) + } + *key = binPath + + return nil +} + +func SetConfigOptionCaptiveCoreConfigDir(co *config.ConfigOption) error { + dirPath := viper.GetString(co.Name) + + fileInfo, err := os.Stat(dirPath) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("captive core configuration files dir %s does not exist", dirPath) + } + + if !fileInfo.IsDir() { + return fmt.Errorf("captive core configuration files dir %s is not a directory", dirPath) + } + + key, ok := co.ConfigKey.(*string) + if !ok { + return unexpectedTypeError(key, co) + } + *key = dirPath + + return nil +} diff --git a/cmd/utils/custom_set_value_test.go b/cmd/utils/custom_set_value_test.go index d390d5c..286d720 100644 --- a/cmd/utils/custom_set_value_test.go +++ b/cmd/utils/custom_set_value_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/stellar/go/support/config" "github.com/stellar/wallet-backend/internal/utils" @@ -124,3 +125,145 @@ func TestSetConfigOptionStellarPublicKey(t *testing.T) { }) } } + +func Test_SetConfigOptionLogLevel(t *testing.T) { + opts := struct{ logrusLevel logrus.Level }{} + + co := config.ConfigOption{ + Name: "log-level", + OptType: types.String, + CustomSetValue: SetConfigOptionLogLevel, + ConfigKey: &opts.logrusLevel, + } + + testCases := []customSetterTestCase[logrus.Level]{ + { + name: "returns an error if the log level is empty", + args: []string{}, + wantErrContains: `couldn't parse log level in log-level: not a valid logrus Level: ""`, + }, + { + name: "returns an error if the log level is invalid", + args: []string{"--log-level", "test"}, + wantErrContains: `couldn't parse log level in log-level: not a valid logrus Level: "test"`, + }, + { + name: "handles messenger type TRACE (through CLI args)", + args: []string{"--log-level", "TRACE"}, + wantResult: logrus.TraceLevel, + }, + { + name: "handles messenger type TRACE (through ENV vars)", + envValue: "TRACE", + wantResult: logrus.TraceLevel, + }, + { + name: "handles messenger type INFO (through CLI args)", + args: []string{"--log-level", "iNfO"}, + wantResult: logrus.InfoLevel, + }, + { + name: "handles messenger type INFO (through ENV vars)", + envValue: "INFO", + wantResult: logrus.InfoLevel, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opts.logrusLevel = 0 + customSetterTester[logrus.Level](t, tc, co) + }) + } +} + +func TestSetConfigOptionCaptiveCoreBinPath(t *testing.T) { + opts := struct{ binPath string }{} + + co := config.ConfigOption{ + Name: "captive-core-bin-path", + OptType: types.String, + CustomSetValue: SetConfigOptionCaptiveCoreBinPath, + ConfigKey: &opts.binPath, + } + + testCases := []customSetterTestCase[string]{ + { + name: "returns an error if the file path is not set, should be caught by the Require() function", + wantErrContains: "binary file does not exist", + }, + { + name: "returns an error if the path is invalid", + args: []string{"--captive-core-bin-path", "/a/random/path/bin"}, + wantErrContains: "binary file /a/random/path/bin does not exist", + }, + { + name: "returns an error if the path format is invalid", + args: []string{"--captive-core-bin-path", "^7JcrS8J4q@V0$c"}, + wantErrContains: "binary file ^7JcrS8J4q@V0$c does not exist", + }, + { + name: "returns an error if the path is a directory, not a file", + args: []string{"--captive-core-bin-path", "./"}, + wantErrContains: "binary file path ./ is a directory, not a file", + }, + { + name: "sets to ENV var value", + envValue: "./custom_set_value_test.go", + wantResult: "./custom_set_value_test.go", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opts.binPath = "" + customSetterTester(t, tc, co) + }) + } +} + +func TestSetConfigOptionCaptiveCoreConfigDir(t *testing.T) { + opts := struct{ binPath string }{} + + co := config.ConfigOption{ + Name: "captive-core-config-dir", + OptType: types.String, + CustomSetValue: SetConfigOptionCaptiveCoreConfigDir, + ConfigKey: &opts.binPath, + } + + testCases := []customSetterTestCase[string]{ + { + name: "returns an error if the file path is not set, should be caught by the Require() function", + wantErrContains: "captive core configuration files dir does not exist", + }, + { + name: "returns an error if the path is invalid", + envValue: "/a/random/path", + wantErrContains: "captive core configuration files dir /a/random/path does not exist", + }, + { + name: "returns an error if the path format is invalid", + envValue: "^7JcrS8J4q@V0$c", + wantErrContains: "captive core configuration files dir ^7JcrS8J4q@V0$c does not exist", + }, + + { + name: "returns an error if the path is a file, not a directory", + envValue: "./custom_set_value_test.go", + wantErrContains: "captive core configuration files dir ./custom_set_value_test.go is not a directory", + }, + { + name: "sets to ENV var value", + envValue: "../../internal/ingest/config", + wantResult: "../../internal/ingest/config", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opts.binPath = "" + customSetterTester(t, tc, co) + }) + } +} diff --git a/go.mod b/go.mod index 1866e67..7656a48 100644 --- a/go.mod +++ b/go.mod @@ -15,24 +15,44 @@ require ( ) require ( + cloud.google.com/go v0.112.0 // indirect + cloud.google.com/go/compute v1.23.3 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.5 // indirect + cloud.google.com/go/storage v1.37.0 // indirect github.com/BurntSushi/toml v1.3.2 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go v1.45.26 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/djherbis/fscache v0.10.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.5.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/schema v1.2.0 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -51,11 +71,32 @@ require ( github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.18.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/mod v0.13.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.14.0 // indirect + google.golang.org/api v0.157.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect + google.golang.org/grpc v1.60.1 // indirect google.golang.org/protobuf v1.32.0 // indirect + gopkg.in/djherbis/atime.v1 v1.0.0 // indirect + gopkg.in/djherbis/stream.v1 v1.3.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/tylerb/graceful.v1 v1.2.15 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 9cff354..db21dc2 100644 --- a/go.sum +++ b/go.sum @@ -17,14 +17,22 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -35,6 +43,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.37.0 h1:WI8CsaFO8Q9KjPVtsZ5Cmi0dXV25zMoX0FklT7c3Jm4= +cloud.google.com/go/storage v1.37.0/go.mod h1:i34TiT2IhiNDmcj65PqwCjcoUX7Z5pLzS8DEmoiFq1k= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= @@ -42,14 +52,20 @@ github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f h1:zvClvFQwU++UpIUBGC8YmDlfhUrweEy1R1Fj1gu5iIM= github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.45.26 h1:PJ2NJNY5N/yeobLYe1Y+xLdavBi67ZI8gvph6ftwVCg= +github.com/aws/aws-sdk-go v1.45.26/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -60,19 +76,27 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/djherbis/fscache v0.10.1 h1:hDv+RGyvD+UDKyRYuLoVNbuRTnf2SrA2K3VyR1br9lk= +github.com/djherbis/fscache v0.10.1/go.mod h1:yyPYtkNnnPXsW+81lAcQS6yab3G2CRfnPLotBvtbf0c= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -88,12 +112,19 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -116,6 +147,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -128,15 +160,19 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v0.0.0-20160401233042-9235644dd9e5 h1:oERTZ1buOUYlpmKaqlO5fYmz8cZ1rYu5DieJzF4ZVmU= github.com/google/go-querystring v0.0.0-20160401233042-9235644dd9e5/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -148,9 +184,17 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/schema v1.2.0 h1:YufUaxZYCKGFuAq3c96BOhjgd5nmXiOY9NGzF247Tsc= github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= @@ -158,6 +202,8 @@ github.com/guregu/null v4.0.0+incompatible h1:4zw0ckM7ECd6FNNddc3Fu4aty9nTlpkkzH github.com/guregu/null v4.0.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -168,6 +214,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jarcoal/httpmock v0.0.0-20161210151336-4442edb3db31 h1:Aw95BEvxJ3K6o9GGv5ppCd1P8hkeIeEJ30FO+OhOJpM= github.com/jarcoal/httpmock v0.0.0-20161210151336-4442edb3db31/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -207,6 +257,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -271,6 +323,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -298,12 +351,27 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -312,7 +380,10 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -348,6 +419,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -376,10 +450,13 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -391,6 +468,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -401,6 +480,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -436,11 +518,16 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -449,11 +536,15 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -501,10 +592,15 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -524,6 +620,8 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.157.0 h1:ORAeqmbrrozeyw5NjnMxh7peHO0UzV4wWYSwZeCUb20= +google.golang.org/api v0.157.0/go.mod h1:+z4v4ufbZ1WEpld6yMGHyggs+PmAHiaLNj5ytP3N01g= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -531,6 +629,8 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -567,6 +667,12 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= +google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457 h1:KHBtwE+eQc3+NxpjmRFlQ3pJQ2FNnhhgB9xOV8kyBuU= +google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -583,6 +689,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -601,6 +709,10 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/djherbis/atime.v1 v1.0.0 h1:eMRqB/JrLKocla2PBPKgQYg/p5UG4L6AUAs92aP7F60= +gopkg.in/djherbis/atime.v1 v1.0.0/go.mod h1:hQIUStKmJfvf7xdh/wtK84qe+DsTV5LnA9lzxxtPpJ8= +gopkg.in/djherbis/stream.v1 v1.3.1 h1:uGfmsOY1qqMjQQphhRBSGLyA9qumJ56exkRu9ASTjCw= +gopkg.in/djherbis/stream.v1 v1.3.1/go.mod h1:aEV8CBVRmSpLamVJfM903Npic1IKmb2qS30VAZ+sssg= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/gavv/httpexpect.v1 v1.0.0-20170111145843-40724cf1e4a0 h1:r5ptJ1tBxVAeqw4CrYWhXIMr0SybY3CDHuIbCg5CFVw= gopkg.in/gavv/httpexpect.v1 v1.0.0-20170111145843-40724cf1e4a0/go.mod h1:WtiW9ZA1LdaWqtQRo1VbIL/v4XZ8NDta+O/kSpGgVek= @@ -611,6 +723,9 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/tylerb/graceful.v1 v1.2.15 h1:1JmOyhKqAyX3BgTXMI84LwT6FOJ4tP2N9e2kwTCM0nQ= gopkg.in/tylerb/graceful.v1 v1.2.15/go.mod h1:yBhekWvR20ACXVObSSdD3u6S9DeSylanL2PAbAC/uJ8= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/data/models.go b/internal/data/models.go index 2da7db6..fbd9c5d 100644 --- a/internal/data/models.go +++ b/internal/data/models.go @@ -16,6 +16,6 @@ func NewModels(db db.ConnectionPool) (*Models, error) { } return &Models{ - Payments: &PaymentModel{db: db}, + Payments: &PaymentModel{DB: db}, }, nil } diff --git a/internal/data/payments.go b/internal/data/payments.go index 4c19ff1..a87bf55 100644 --- a/internal/data/payments.go +++ b/internal/data/payments.go @@ -2,18 +2,99 @@ package data import ( "context" + "database/sql" "fmt" + "time" "github.com/stellar/wallet-backend/internal/db" ) type PaymentModel struct { - db db.ConnectionPool + DB db.ConnectionPool +} + +type Payment struct { + OperationID int64 `db:"operation_id"` + OperationType string `db:"operation_type"` + TransactionID int64 `db:"transaction_id"` + TransactionHash string `db:"transaction_hash"` + FromAddress string `db:"from_address"` + ToAddress string `db:"to_address"` + SrcAssetCode string `db:"src_asset_code"` + SrcAssetIssuer string `db:"src_asset_issuer"` + SrcAmount int64 `db:"src_amount"` + DestAssetCode string `db:"dest_asset_code"` + DestAssetIssuer string `db:"dest_asset_issuer"` + DestAmount int64 `db:"dest_amount"` + CreatedAt time.Time `db:"created_at"` + Memo *string `db:"memo"` +} + +func (m *PaymentModel) GetLatestLedgerSynced(ctx context.Context, cursorName string) (uint32, error) { + var lastSyncedLedger uint32 + err := m.DB.QueryRowxContext(ctx, `SELECT value FROM ingest_store WHERE key = $1`, cursorName).Scan(&lastSyncedLedger) + // First run, key does not exist yet + if err == sql.ErrNoRows { + return 0, nil + } + if err != nil { + return 0, fmt.Errorf("getting latest ledger synced for cursor %s: %w", cursorName, err) + } + + return lastSyncedLedger, nil +} + +func (m *PaymentModel) UpdateLatestLedgerSynced(ctx context.Context, cursorName string, ledger uint32) error { + const query = ` + INSERT INTO ingest_store (key, value) VALUES ($1, $2) + ON CONFLICT (key) DO UPDATE SET value = excluded.value + ` + _, err := m.DB.ExecContext(ctx, query, cursorName, ledger) + if err != nil { + return fmt.Errorf("updating last synced ledger to %d: %w", ledger, err) + } + + return nil +} + +func (m *PaymentModel) AddPayment(ctx context.Context, tx db.Transaction, payment Payment) error { + const query = ` + INSERT INTO ingest_payments ( + operation_id, operation_type, transaction_id, transaction_hash, from_address, to_address, src_asset_code, src_asset_issuer, src_amount, + dest_asset_code, dest_asset_issuer, dest_amount, created_at, memo + ) + SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14 + WHERE EXISTS ( + SELECT 1 FROM accounts WHERE stellar_address IN ($5, $6) + ) + ON CONFLICT (operation_id) DO UPDATE SET + operation_type = EXCLUDED.operation_type, + transaction_id = EXCLUDED.transaction_id, + transaction_hash = EXCLUDED.transaction_hash, + from_address = EXCLUDED.from_address, + to_address = EXCLUDED.to_address, + src_asset_code = EXCLUDED.src_asset_code, + src_asset_issuer = EXCLUDED.src_asset_issuer, + src_amount = EXCLUDED.src_amount, + dest_asset_code = EXCLUDED.dest_asset_code, + dest_asset_issuer = EXCLUDED.dest_asset_issuer, + dest_amount = EXCLUDED.dest_amount, + created_at = EXCLUDED.created_at, + memo = EXCLUDED.memo + ; + ` + _, err := tx.ExecContext(ctx, query, payment.OperationID, payment.OperationType, payment.TransactionID, payment.TransactionHash, payment.FromAddress, payment.ToAddress, payment.SrcAssetCode, payment.SrcAssetIssuer, payment.SrcAmount, + payment.DestAssetCode, payment.DestAssetIssuer, payment.DestAmount, payment.CreatedAt, payment.Memo) + if err != nil { + return fmt.Errorf("inserting payment: %w", err) + } + + return nil } func (m *PaymentModel) SubscribeAddress(ctx context.Context, address string) error { const query = `INSERT INTO accounts (stellar_address) VALUES ($1) ON CONFLICT DO NOTHING` - _, err := m.db.ExecContext(ctx, query, address) + _, err := m.DB.ExecContext(ctx, query, address) if err != nil { return fmt.Errorf("subscribing address %s to payments tracking: %w", address, err) } @@ -23,7 +104,7 @@ func (m *PaymentModel) SubscribeAddress(ctx context.Context, address string) err func (m *PaymentModel) UnsubscribeAddress(ctx context.Context, address string) error { const query = `DELETE FROM accounts WHERE stellar_address = $1` - _, err := m.db.ExecContext(ctx, query, address) + _, err := m.DB.ExecContext(ctx, query, address) if err != nil { return fmt.Errorf("unsubscribing address %s to payments tracking: %w", address, err) } diff --git a/internal/data/payments_test.go b/internal/data/payments_test.go index dd7a44b..ff866c8 100644 --- a/internal/data/payments_test.go +++ b/internal/data/payments_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "testing" + "time" "github.com/stellar/go/keypair" "github.com/stellar/wallet-backend/internal/db" @@ -12,6 +13,94 @@ import ( "github.com/stretchr/testify/require" ) +func TestAddPayment(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + m := &PaymentModel{ + DB: dbConnectionPool, + } + ctx := context.Background() + + const ( + fromAddress = "GCYQBVCREYSLKHHOWLT27VNZNGVIXXAPYVNNOWMQV67WVDD4PP2VZAX7" + toAddress = "GDDEAH46MNFO6JD7NTQ5FWJBC4ZSA47YEK3RKFHQWADYTS6NDVD5CORW" + ) + payment := Payment{ + OperationID: 2120562792996865, + OperationType: "OperationTypePayment", + TransactionID: 2120562792996864, + TransactionHash: "a3daffa64dc46db84888b1206dc8014a480042e7fe8b19fd5d05465709f4e887", + FromAddress: fromAddress, + ToAddress: toAddress, + SrcAssetCode: "USDC", + SrcAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5", + SrcAmount: 500000000, + DestAssetCode: "USDC", + DestAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5", + DestAmount: 500000000, + CreatedAt: time.Date(2023, 12, 15, 1, 0, 0, 0, time.UTC), + Memo: nil, + } + + addPayment := func() { + err := db.RunInTransaction(ctx, m.DB, nil, func(dbTx db.Transaction) error { + return m.AddPayment(ctx, dbTx, payment) + }) + require.NoError(t, err) + } + + fetchPaymentInserted := func() bool { + var inserted bool + err := dbConnectionPool.QueryRowxContext(ctx, "SELECT EXISTS(SELECT 1 FROM ingest_payments)").Scan(&inserted) + require.NoError(t, err) + + return inserted + } + + cleanUpDB := func() { + _, err := dbConnectionPool.ExecContext(ctx, `DELETE FROM accounts; DELETE FROM ingest_payments;`) + require.NoError(t, err) + } + + t.Run("unkown_address", func(t *testing.T) { + addPayment() + + inserted := fetchPaymentInserted() + assert.False(t, inserted) + + cleanUpDB() + }) + + t.Run("from_known_address", func(t *testing.T) { + _, err := dbConnectionPool.ExecContext(ctx, `INSERT INTO accounts (stellar_address) VALUES ($1)`, fromAddress) + require.NoError(t, err) + + addPayment() + + inserted := fetchPaymentInserted() + assert.True(t, inserted) + + cleanUpDB() + }) + + t.Run("to_known_address", func(t *testing.T) { + _, err := dbConnectionPool.ExecContext(ctx, `INSERT INTO accounts (stellar_address) VALUES ($1)`, toAddress) + require.NoError(t, err) + + addPayment() + + inserted := fetchPaymentInserted() + assert.True(t, inserted) + + cleanUpDB() + }) +} + func TestSubscribeAddress(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -21,7 +110,7 @@ func TestSubscribeAddress(t *testing.T) { defer dbConnectionPool.Close() m := &PaymentModel{ - db: dbConnectionPool, + DB: dbConnectionPool, } ctx := context.Background() @@ -30,7 +119,7 @@ func TestSubscribeAddress(t *testing.T) { require.NoError(t, err) var dbAddress sql.NullString - err = m.db.GetContext(ctx, &dbAddress, "SELECT stellar_address FROM accounts LIMIT 1") + err = m.DB.GetContext(ctx, &dbAddress, "SELECT stellar_address FROM accounts LIMIT 1") require.NoError(t, err) assert.True(t, dbAddress.Valid) diff --git a/internal/db/db.go b/internal/db/db.go index 87fb273..264aaa8 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -2,27 +2,27 @@ package db import ( "context" + "database/sql" "fmt" "time" "github.com/jmoiron/sqlx" + "github.com/stellar/go/support/log" ) type ConnectionPool interface { + SQLExecuter + BeginTxx(ctx context.Context, opts *sql.TxOptions) (Transaction, error) Close() error - Ping() error - DriverName() string - sqlx.ExecerContext - sqlx.QueryerContext - sqlx.PreparerContext - GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error - SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error + Ping(ctx context.Context) error + SqlDB(ctx context.Context) (*sql.DB, error) + SqlxDB(ctx context.Context) (*sqlx.DB, error) } // Make sure *DBConnectionPoolImplementation implements DBConnectionPool: -var _ ConnectionPool = (*DBConnectionPoolImplementation)(nil) +var _ ConnectionPool = (*ConnectionPoolImplementation)(nil) -type DBConnectionPoolImplementation struct { +type ConnectionPoolImplementation struct { *sqlx.DB } @@ -44,5 +44,97 @@ func OpenDBConnectionPool(dataSourceName string) (ConnectionPool, error) { return nil, fmt.Errorf("error pinging app DB connection pool: %w", err) } - return &DBConnectionPoolImplementation{DB: sqlxDB}, nil + return &ConnectionPoolImplementation{DB: sqlxDB}, nil +} + +func (db *ConnectionPoolImplementation) BeginTxx(ctx context.Context, opts *sql.TxOptions) (Transaction, error) { + return db.DB.BeginTxx(ctx, opts) +} + +func (db *ConnectionPoolImplementation) Ping(ctx context.Context) error { + return db.DB.PingContext(ctx) +} + +func (db *ConnectionPoolImplementation) SqlDB(ctx context.Context) (*sql.DB, error) { + return db.DB.DB, nil +} + +func (db *ConnectionPoolImplementation) SqlxDB(ctx context.Context) (*sqlx.DB, error) { + return db.DB, nil +} + +// Transaction is an interface that wraps the sqlx.Tx structs methods. +type Transaction interface { + SQLExecuter + Rollback() error + Commit() error +} + +// Make sure *sqlx.Tx implements DBTransaction: +var _ Transaction = (*sqlx.Tx)(nil) + +// SQLExecuter is an interface that wraps the *sqlx.DB and *sqlx.Tx structs methods. +type SQLExecuter interface { + DriverName() string + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) + GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error + sqlx.PreparerContext + sqlx.QueryerContext + Rebind(query string) string + SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error +} + +// Make sure *sqlx.DB implements SQLExecuter: +var _ SQLExecuter = (*sqlx.DB)(nil) + +// Make sure DBConnectionPool implements SQLExecuter: +var _ SQLExecuter = (ConnectionPool)(nil) + +// Make sure *sqlx.Tx implements SQLExecuter: +var _ SQLExecuter = (*sqlx.Tx)(nil) + +// Make sure DBTransaction implements SQLExecuter: +var _ SQLExecuter = (Transaction)(nil) + +// RunInTransaction runs the given atomic function in an atomic database transaction and returns an error. Boilerplate +// code for database transactions. +func RunInTransaction(ctx context.Context, dbConnectionPool ConnectionPool, opts *sql.TxOptions, atomicFunction func(dbTx Transaction) error) error { + // wrap the atomic function with a function that returns nil and an error so we can call RunInTransactionWithResult + wrappedFunction := func(dbTx Transaction) (interface{}, error) { + return nil, atomicFunction(dbTx) + } + + _, err := RunInTransactionWithResult(ctx, dbConnectionPool, opts, wrappedFunction) + return err +} + +// RunInTransactionWithResult runs the given atomic function in an atomic database transaction and returns a result and +// an error. Boilerplate code for database transactions. +func RunInTransactionWithResult[T any](ctx context.Context, dbConnectionPool ConnectionPool, opts *sql.TxOptions, atomicFunction func(dbTx Transaction) (T, error)) (result T, err error) { + dbTx, err := dbConnectionPool.BeginTxx(ctx, opts) + if err != nil { + return *new(T), fmt.Errorf("creating db transaction for RunInTransactionWithResult: %w", err) + } + + defer func() { + if err != nil { + log.Ctx(ctx).Errorf("Rolling back transaction due to error: %v", err) + errRollBack := dbTx.Rollback() + if errRollBack != nil { + log.Ctx(ctx).Errorf("Error in database transaction rollback: %v", errRollBack) + } + } + }() + + result, err = atomicFunction(dbTx) + if err != nil { + return *new(T), fmt.Errorf("running atomic function in RunInTransactionWithResult: %w", err) + } + + err = dbTx.Commit() + if err != nil { + return *new(T), fmt.Errorf("committing transaction in RunInTransactionWithResult: %w", err) + } + + return result, nil } diff --git a/internal/db/db_test.go b/internal/db/db_test.go index ee64545..d2cbb2f 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -1,6 +1,7 @@ package db import ( + "context" "testing" "github.com/stellar/go/support/db/dbtest" @@ -18,6 +19,7 @@ func TestOpenDBConnectionPool(t *testing.T) { assert.Equal(t, "postgres", dbConnectionPool.DriverName()) - err = dbConnectionPool.Ping() + ctx := context.Background() + err = dbConnectionPool.Ping(ctx) require.NoError(t, err) } diff --git a/internal/db/migrations/2024-05-22.0-ingest_payments.sql b/internal/db/migrations/2024-05-22.0-ingest_payments.sql new file mode 100644 index 0000000..3f4bea4 --- /dev/null +++ b/internal/db/migrations/2024-05-22.0-ingest_payments.sql @@ -0,0 +1,34 @@ +-- +migrate Up + +CREATE TABLE ingest_store ( + key varchar(255) NOT NULL, + value varchar(255) NOT NULL, + PRIMARY KEY (key) +); + +CREATE TABLE ingest_payments ( + operation_id bigint NOT NULL, + operation_type text NOT NULL, + transaction_id bigint NOT NULL, + transaction_hash text NOT NULL, + from_address text NOT NULL, + to_address text NOT NULL, + src_asset_code text NOT NULL, + src_asset_issuer text NOT NULL, + src_amount bigint NOT NULL, + dest_asset_code text NOT NULL, + dest_asset_issuer text NOT NULL, + dest_amount bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + memo text, + PRIMARY KEY (operation_id) +); + +CREATE INDEX from_address_idx ON ingest_payments (from_address); +CREATE INDEX to_address_idx ON ingest_payments (to_address); + +-- +migrate Down + +DROP TABLE ingest_payments; + +DROP TABLE ingest_store; diff --git a/internal/ingest/config/stellar-core_pubnet.cfg b/internal/ingest/config/stellar-core_pubnet.cfg new file mode 100644 index 0000000..8394e87 --- /dev/null +++ b/internal/ingest/config/stellar-core_pubnet.cfg @@ -0,0 +1,25 @@ +# Stellar Pubnet validators +[[HOME_DOMAINS]] +HOME_DOMAIN="www.stellar.org" +QUALITY="HIGH" + +[[VALIDATORS]] +NAME="SDF 1" +PUBLIC_KEY="GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH" +ADDRESS="core-live-a.stellar.org:11625" +HISTORY="curl -sf http://history.stellar.org/prd/core-live/core_live_001/{0} -o {1}" +HOME_DOMAIN="www.stellar.org" + +[[VALIDATORS]] +NAME="SDF 2" +PUBLIC_KEY="GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK" +ADDRESS="core-live-b.stellar.org:11625" +HISTORY="curl -sf http://history.stellar.org/prd/core-live/core_live_002/{0} -o {1}" +HOME_DOMAIN="www.stellar.org" + +[[VALIDATORS]] +NAME="SDF 3" +PUBLIC_KEY="GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ" +ADDRESS="core-live-c.stellar.org:11625" +HISTORY="curl -sf http://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" +HOME_DOMAIN="www.stellar.org" \ No newline at end of file diff --git a/internal/ingest/config/stellar-core_testnet.cfg b/internal/ingest/config/stellar-core_testnet.cfg new file mode 100644 index 0000000..357470a --- /dev/null +++ b/internal/ingest/config/stellar-core_testnet.cfg @@ -0,0 +1,25 @@ +# Stellar Testnet validators +[[HOME_DOMAINS]] +HOME_DOMAIN="testnet.stellar.org" +QUALITY="HIGH" + +[[VALIDATORS]] +NAME="sdftest1" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" +ADDRESS="core-testnet1.stellar.org" +HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_001/{0} -o {1}" + +[[VALIDATORS]] +NAME="sdftest2" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GCUCJTIYXSOXKBSNFGNFWW5MUQ54HKRPGJUTQFJ5RQXZXNOLNXYDHRAP" +ADDRESS="core-testnet2.stellar.org" +HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_002/{0} -o {1}" + +[[VALIDATORS]] +NAME="sdftest3" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GC2V2EFSXN6SQTWVYA5EPJPBWWIMSD2XQNKUOHGEKB535AQE2I6IXV2Z" +ADDRESS="core-testnet3.stellar.org" +HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_003/{0} -o {1}" \ No newline at end of file diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go new file mode 100644 index 0000000..33a6b88 --- /dev/null +++ b/internal/ingest/ingest.go @@ -0,0 +1,116 @@ +package ingest + +import ( + "context" + "errors" + "fmt" + "os" + "path" + + "github.com/sirupsen/logrus" + "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/network" + "github.com/stellar/go/support/log" + "github.com/stellar/wallet-backend/internal/data" + "github.com/stellar/wallet-backend/internal/db" + "github.com/stellar/wallet-backend/internal/services" +) + +type Configs struct { + DatabaseURL string + NetworkPassphrase string + CaptiveCoreBinPath string + CaptiveCoreConfigDir string + LedgerCursorName string + StartLedger int + EndLedger int + LogLevel logrus.Level +} + +func Ingest(cfg Configs) error { + ctx := context.Background() + + manager, err := setupDeps(cfg) + if err != nil { + log.Ctx(ctx).Fatalf("Error setting up dependencies for ingest: %v", err) + } + + if err = manager.Run(ctx, uint32(cfg.StartLedger), uint32(cfg.EndLedger)); err != nil { + log.Ctx(ctx).Fatalf("Running ingest from %d to %d: %v", cfg.StartLedger, cfg.EndLedger, err) + } + + return nil +} + +func setupDeps(cfg Configs) (*services.IngestManager, error) { + // Open DB connection pool + dbConnectionPool, err := db.OpenDBConnectionPool(cfg.DatabaseURL) + if err != nil { + return nil, fmt.Errorf("error connecting to the database: %w", err) + } + models, err := data.NewModels(dbConnectionPool) + if err != nil { + return nil, fmt.Errorf("error creating models for Serve: %w", err) + } + + // Setup Captive Core backend + captiveCoreConfig, err := getCaptiveCoreConfig(cfg) + if err != nil { + return nil, fmt.Errorf("getting captive core config: %w", err) + } + ledgerBackend, err := ledgerbackend.NewCaptive(captiveCoreConfig) + if err != nil { + return nil, fmt.Errorf("creating captive core backend: %w", err) + } + + return &services.IngestManager{ + NetworkPassphrase: cfg.NetworkPassphrase, + LedgerCursorName: cfg.LedgerCursorName, + LedgerBackend: ledgerBackend, + PaymentModel: models.Payments, + }, nil +} + +const ( + configFileNamePubnet = "stellar-core_pubnet.cfg" + configFileNameTestnet = "stellar-core_testnet.cfg" +) + +func getCaptiveCoreConfig(cfg Configs) (ledgerbackend.CaptiveCoreConfig, error) { + var networkArchivesURLs []string + var configFilePath string + + switch cfg.NetworkPassphrase { + case network.TestNetworkPassphrase: + networkArchivesURLs = network.TestNetworkhistoryArchiveURLs + configFilePath = path.Join(cfg.CaptiveCoreConfigDir, configFileNameTestnet) + case network.PublicNetworkPassphrase: + networkArchivesURLs = network.PublicNetworkhistoryArchiveURLs + configFilePath = path.Join(cfg.CaptiveCoreConfigDir, configFileNamePubnet) + default: + return ledgerbackend.CaptiveCoreConfig{}, fmt.Errorf("unknown network: %s", cfg.NetworkPassphrase) + } + + if _, err := os.Stat(configFilePath); errors.Is(err, os.ErrNotExist) { + return ledgerbackend.CaptiveCoreConfig{}, fmt.Errorf("captive core configuration file not found in %s", configFilePath) + } + + // Read configuration TOML + captiveCoreToml, err := ledgerbackend.NewCaptiveCoreTomlFromFile(configFilePath, ledgerbackend.CaptiveCoreTomlParams{ + CoreBinaryPath: cfg.CaptiveCoreBinPath, + NetworkPassphrase: cfg.NetworkPassphrase, + HistoryArchiveURLs: networkArchivesURLs, + UseDB: true, + }) + if err != nil { + return ledgerbackend.CaptiveCoreConfig{}, fmt.Errorf("creating captive core toml: %w", err) + } + + return ledgerbackend.CaptiveCoreConfig{ + NetworkPassphrase: cfg.NetworkPassphrase, + HistoryArchiveURLs: networkArchivesURLs, + BinaryPath: cfg.CaptiveCoreBinPath, + Toml: captiveCoreToml, + UseDB: true, + }, nil +} diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go new file mode 100644 index 0000000..5bf4c59 --- /dev/null +++ b/internal/ingest/ingest_test.go @@ -0,0 +1,61 @@ +package ingest + +import ( + "testing" + + "github.com/stellar/go/network" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetCaptiveCoreConfig(t *testing.T) { + t.Run("testnet_success", func(t *testing.T) { + config, err := getCaptiveCoreConfig(Configs{ + NetworkPassphrase: network.TestNetworkPassphrase, + CaptiveCoreBinPath: "/bin/path", + CaptiveCoreConfigDir: "./config", + }) + + require.NoError(t, err) + assert.Equal(t, "/bin/path", config.BinaryPath) + assert.Equal(t, network.TestNetworkPassphrase, config.NetworkPassphrase) + assert.Equal(t, network.TestNetworkhistoryArchiveURLs, config.HistoryArchiveURLs) + assert.Equal(t, true, config.UseDB) + assert.NotNil(t, config.Toml) + }) + + t.Run("pubnet_success", func(t *testing.T) { + config, err := getCaptiveCoreConfig(Configs{ + NetworkPassphrase: network.PublicNetworkPassphrase, + CaptiveCoreBinPath: "/bin/path", + CaptiveCoreConfigDir: "./config", + }) + + require.NoError(t, err) + assert.Equal(t, "/bin/path", config.BinaryPath) + assert.Equal(t, network.PublicNetworkPassphrase, config.NetworkPassphrase) + assert.Equal(t, network.PublicNetworkhistoryArchiveURLs, config.HistoryArchiveURLs) + assert.Equal(t, true, config.UseDB) + assert.NotNil(t, config.Toml) + }) + + t.Run("unknown_network", func(t *testing.T) { + _, err := getCaptiveCoreConfig(Configs{ + NetworkPassphrase: "Invalid SDF Network ; May 2024", + CaptiveCoreBinPath: "/bin/path", + CaptiveCoreConfigDir: "./config", + }) + + assert.ErrorContains(t, err, "unknown network: Invalid SDF Network ; May 2024") + }) + + t.Run("invalid_config_file", func(t *testing.T) { + _, err := getCaptiveCoreConfig(Configs{ + NetworkPassphrase: network.TestNetworkPassphrase, + CaptiveCoreBinPath: "/bin/path", + CaptiveCoreConfigDir: "./invalid/path", + }) + + assert.ErrorContains(t, err, "captive core configuration file not found in invalid/path/stellar-core_testnet.cfg") + }) +} diff --git a/internal/serve/serve.go b/internal/serve/serve.go index fa9e738..a2d2326 100644 --- a/internal/serve/serve.go +++ b/internal/serve/serve.go @@ -5,8 +5,9 @@ import ( "net/http" "github.com/go-chi/chi" + "github.com/sirupsen/logrus" supporthttp "github.com/stellar/go/support/http" - supportlog "github.com/stellar/go/support/log" + "github.com/stellar/go/support/log" "github.com/stellar/go/support/render/health" "github.com/stellar/wallet-backend/internal/data" "github.com/stellar/wallet-backend/internal/db" @@ -17,16 +18,17 @@ import ( ) type Configs struct { - Logger *supportlog.Entry Port int + DatabaseURL string ServerBaseURL string WalletSigningKey string - DatabaseURL string + LogLevel logrus.Level } type handlerDeps struct { - Logger *supportlog.Entry Models *data.Models + Port int + DatabaseURL string SignatureVerifier auth.SignatureVerifier } @@ -41,10 +43,10 @@ func Serve(cfg Configs) error { ListenAddr: addr, Handler: handler(deps), OnStarting: func() { - deps.Logger.Infof("Starting Wallet Backend server on %s", addr) + log.Infof("Starting Wallet Backend server on port %d", cfg.Port) }, OnStopping: func() { - deps.Logger.Info("Stopping Wallet Backend server") + log.Info("Stopping Wallet Backend server") }, }) @@ -67,14 +69,13 @@ func getHandlerDeps(cfg Configs) (handlerDeps, error) { } return handlerDeps{ - Logger: cfg.Logger, Models: models, SignatureVerifier: signatureVerifier, }, nil } func handler(deps handlerDeps) http.Handler { - mux := supporthttp.NewAPIMux(deps.Logger) + mux := supporthttp.NewAPIMux(log.DefaultLogger) mux.NotFound(httperror.ErrorHandler{Error: httperror.NotFound}.ServeHTTP) mux.MethodNotAllowed(httperror.ErrorHandler{Error: httperror.MethodNotAllowed}.ServeHTTP) diff --git a/internal/services/ingest.go b/internal/services/ingest.go new file mode 100644 index 0000000..0c9457a --- /dev/null +++ b/internal/services/ingest.go @@ -0,0 +1,220 @@ +package services + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + "github.com/stellar/go/ingest" + "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/support/log" + "github.com/stellar/go/xdr" + "github.com/stellar/wallet-backend/internal/data" + "github.com/stellar/wallet-backend/internal/db" + "github.com/stellar/wallet-backend/internal/utils" +) + +type IngestManager struct { + PaymentModel *data.PaymentModel + LedgerBackend ledgerbackend.LedgerBackend + NetworkPassphrase string + LedgerCursorName string +} + +func (m *IngestManager) Run(ctx context.Context, start, end uint32) error { + var ingestLedger uint32 + + if start == 0 { + lastSyncedLedger, err := m.PaymentModel.GetLatestLedgerSynced(ctx, m.LedgerCursorName) + if err != nil { + return fmt.Errorf("getting last ledger synced: %w", err) + } + + if lastSyncedLedger == 0 { + return errors.New("ingestion service is not initialized, --start flag is required") + } + ingestLedger = lastSyncedLedger + } else { + ingestLedger = start + } + + if end != 0 && ingestLedger > end { + return fmt.Errorf("starting ledger (%d) may not be greater than ending ledger (%d)", ingestLedger, end) + } + + err := m.maybePrepareRange(ctx, ingestLedger, end) + if err != nil { + return fmt.Errorf("preparing range from %d to %d: %w", ingestLedger, end, err) + } + + heartbeat := make(chan any) + go trackServiceHealth(heartbeat) + + for ; end == 0 || ingestLedger <= end; ingestLedger++ { + log.Ctx(ctx).Infof("waiting for ledger %d", ingestLedger) + + ledgerMeta, err := m.LedgerBackend.GetLedger(ctx, ingestLedger) + if err != nil { + return fmt.Errorf("getting ledger meta for ledger %d: %w", ingestLedger, err) + } + + heartbeat <- true + + err = m.processLedger(ctx, ingestLedger, ledgerMeta) + if err != nil { + return fmt.Errorf("processing ledger %d: %w", ingestLedger, err) + } + + log.Ctx(ctx).Infof("ledger %d successfully processed", ingestLedger) + } + + return nil +} + +func (m *IngestManager) maybePrepareRange(ctx context.Context, from, to uint32) error { + var ledgerRange ledgerbackend.Range + if to == 0 { + ledgerRange = ledgerbackend.UnboundedRange(from) + } else { + ledgerRange = ledgerbackend.BoundedRange(from, to) + } + + prepared, err := m.LedgerBackend.IsPrepared(ctx, ledgerRange) + if err != nil { + return fmt.Errorf("checking prepared range: %w", err) + } + + if !prepared { + err = m.LedgerBackend.PrepareRange(ctx, ledgerRange) + if err != nil { + return fmt.Errorf("preparing range: %w", err) + } + } + + return nil +} + +func trackServiceHealth(heartbeat chan any) { + const alertAfter = time.Second * 60 + ticker := time.NewTicker(alertAfter) + + for { + select { + case <-ticker.C: + warn := fmt.Sprintf("ingestion service stale for over %s", alertAfter) + log.Warn(warn) + // TODO: track in Sentry + // sentry.CaptureMessage(warn) + ticker.Reset(alertAfter) + case <-heartbeat: + ticker.Reset(alertAfter) + } + } +} + +func (m *IngestManager) processLedger(ctx context.Context, ledger uint32, ledgerMeta xdr.LedgerCloseMeta) (err error) { + reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(m.NetworkPassphrase, ledgerMeta) + if err != nil { + return fmt.Errorf("creating ledger reader: %w", err) + } + + ledgerCloseTime := time.Unix(int64(ledgerMeta.LedgerHeaderHistoryEntry().Header.ScpValue.CloseTime), 0).UTC() + ledgerSequence := ledgerMeta.LedgerSequence() + + return db.RunInTransaction(ctx, m.PaymentModel.DB, nil, func(dbTx db.Transaction) error { + for { + tx, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("reading transaction: %w", err) + } + + if !tx.Result.Successful() { + continue + } + + txHash := utils.TransactionHash(ledgerMeta, int(tx.Index)) + txMemo := utils.Memo(tx.Envelope.Memo(), txHash) + // The memo field is subject to user input, so we sanitize before persisting in the database + if txMemo != nil { + *txMemo = utils.SanitizeUTF8(*txMemo) + } + + for idx, op := range tx.Envelope.Operations() { + opIdx := idx + 1 + + payment := data.Payment{ + OperationID: utils.OperationID(int32(ledgerSequence), int32(tx.Index), int32(opIdx)), + OperationType: op.Body.Type.String(), + TransactionID: utils.TransactionID(int32(ledgerSequence), int32(tx.Index)), + TransactionHash: txHash, + FromAddress: utils.SourceAccount(op, tx), + CreatedAt: ledgerCloseTime, + Memo: txMemo, + } + + switch op.Body.Type { + case xdr.OperationTypePayment: + fillPayment(&payment, op.Body) + case xdr.OperationTypePathPaymentStrictSend: + fillPathSend(&payment, op.Body, tx, opIdx) + case xdr.OperationTypePathPaymentStrictReceive: + fillPathReceive(&payment, op.Body, tx, opIdx) + default: + continue + } + + err = m.PaymentModel.AddPayment(ctx, dbTx, payment) + if err != nil { + return fmt.Errorf("adding payment for ledger %d, tx %q (%d), operation %d (%d): %w", ledgerSequence, txHash, tx.Index, payment.OperationID, opIdx, err) + } + } + } + + err = m.PaymentModel.UpdateLatestLedgerSynced(ctx, m.LedgerCursorName, ledger) + if err != nil { + return err + } + + return nil + }) +} + +func fillPayment(payment *data.Payment, operation xdr.OperationBody) { + paymentOp := operation.MustPaymentOp() + payment.ToAddress = paymentOp.Destination.Address() + payment.SrcAssetCode = utils.AssetCode(paymentOp.Asset) + payment.DestAssetCode = payment.SrcAssetCode + payment.SrcAssetIssuer = paymentOp.Asset.GetIssuer() + payment.DestAssetIssuer = payment.SrcAssetIssuer + payment.SrcAmount = int64(paymentOp.Amount) + payment.DestAmount = int64(paymentOp.Amount) +} + +func fillPathSend(payment *data.Payment, operation xdr.OperationBody, transaction ingest.LedgerTransaction, operationIdx int) { + pathOp := operation.MustPathPaymentStrictSendOp() + result := utils.OperationResult(transaction, operationIdx).MustPathPaymentStrictSendResult() + payment.ToAddress = pathOp.Destination.Address() + payment.SrcAssetCode = utils.AssetCode(pathOp.SendAsset) + payment.DestAssetCode = utils.AssetCode(pathOp.DestAsset) + payment.SrcAssetIssuer = pathOp.SendAsset.GetIssuer() + payment.DestAssetIssuer = pathOp.DestAsset.GetIssuer() + payment.SrcAmount = int64(pathOp.SendAmount) + payment.DestAmount = int64(result.DestAmount()) +} + +func fillPathReceive(payment *data.Payment, operation xdr.OperationBody, transaction ingest.LedgerTransaction, operationIdx int) { + pathOp := operation.MustPathPaymentStrictReceiveOp() + result := utils.OperationResult(transaction, operationIdx).MustPathPaymentStrictReceiveResult() + payment.ToAddress = pathOp.Destination.Address() + payment.SrcAssetCode = utils.AssetCode(pathOp.SendAsset) + payment.DestAssetCode = utils.AssetCode(pathOp.DestAsset) + payment.SrcAssetIssuer = pathOp.SendAsset.GetIssuer() + payment.DestAssetIssuer = pathOp.DestAsset.GetIssuer() + payment.SrcAmount = int64(result.SendAmount()) + payment.DestAmount = int64(pathOp.DestAmount) +} diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go new file mode 100644 index 0000000..353e0e0 --- /dev/null +++ b/internal/services/ingest_test.go @@ -0,0 +1,144 @@ +package services + +import ( + "context" + "testing" + "time" + + "github.com/stellar/go/network" + "github.com/stellar/go/xdr" + "github.com/stellar/wallet-backend/internal/data" + "github.com/stellar/wallet-backend/internal/db" + "github.com/stellar/wallet-backend/internal/db/dbtest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestProcessLedger(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + m := &IngestManager{ + PaymentModel: &data.PaymentModel{ + DB: dbConnectionPool, + }, + NetworkPassphrase: network.TestNetworkPassphrase, + LedgerCursorName: "last_synced_ledger", + LedgerBackend: nil, + } + + ctx := context.Background() + + // Insert destination account into subscribed addresses + destinationAccount := "GBLI2OE4H3HAW7Z2GXLYZQNQ57XLHJ5OILFPVL33EPA4GDAIQ5F33JGA" + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", destinationAccount) + require.NoError(t, err) + + ledgerMeta := xdr.LedgerCloseMeta{ + V: 1, + V1: &xdr.LedgerCloseMetaV1{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: 123, + ScpValue: xdr.StellarValue{ + CloseTime: xdr.TimePoint(time.Date(2024, 5, 28, 11, 0, 0, 0, time.UTC).Unix()), + }, + LedgerVersion: 10, + }, + }, + TxSet: xdr.GeneralizedTransactionSet{ + V1TxSet: &xdr.TransactionSetV1{ + Phases: []xdr.TransactionPhase{ + { + V0Components: &[]xdr.TxSetComponent{ + { + TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ + Txs: []xdr.TransactionEnvelope{ + { + Type: xdr.EnvelopeTypeEnvelopeTypeTx, + V1: &xdr.TransactionV1Envelope{ + Tx: xdr.Transaction{ + SourceAccount: xdr.MustMuxedAddress("GB3H2CRRTO7W5WF54K53A3MRAFEUISHZ7Y5YGRVGRGHUZESLV5VYYWXI"), + SeqNum: 321, + Memo: xdr.MemoText("memo_test"), + Operations: []xdr.Operation{ + { + SourceAccount: nil, + Body: xdr.OperationBody{ + Type: xdr.OperationTypePayment, + PaymentOp: &xdr.PaymentOp{ + Destination: xdr.MustMuxedAddress(destinationAccount), + Asset: xdr.Asset{ + Type: xdr.AssetTypeAssetTypeCreditAlphanum4, + AlphaNum4: &xdr.AlphaNum4{ + AssetCode: xdr.AssetCode4([]byte("USDC")), + Issuer: xdr.MustMuxedAddress("GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5").ToAccountId(), + }, + }, + Amount: xdr.Int64(50), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + TxProcessing: []xdr.TransactionResultMeta{ + { + Result: xdr.TransactionResultPair{ + TransactionHash: xdr.Hash{}, + }, + TxApplyProcessing: xdr.TransactionMeta{ + V: 3, + }, + }, + }, + }, + } + + // Compute transaction hash and inject into ledger meta + components := ledgerMeta.V1.TxSet.V1TxSet.Phases[0].V0Components + xdrHash, err := network.HashTransactionInEnvelope((*components)[0].TxsMaybeDiscountedFee.Txs[0], m.NetworkPassphrase) + require.NoError(t, err) + ledgerMeta.V1.TxProcessing[0].Result.TransactionHash = xdrHash + + // Run ledger ingestion + err = m.processLedger(ctx, 1, ledgerMeta) + require.NoError(t, err) + + // Assert payment properly persisted to database + var payment data.Payment + query := `SELECT operation_id, operation_type, transaction_id, transaction_hash, from_address, to_address, src_asset_code, src_asset_issuer, src_amount, dest_asset_code, dest_asset_issuer, dest_amount, created_at, memo FROM ingest_payments` + err = dbConnectionPool.GetContext(ctx, &payment, query) + require.NoError(t, err) + + expectedMemo := "memo_test" + assert.Equal(t, data.Payment{ + OperationID: 528280981505, + OperationType: "OperationTypePayment", + TransactionID: 528280981504, + TransactionHash: "c20936e363c85799b31fd321b67aa49ecd88f04fc41297959387e445245080db", + FromAddress: "GB3H2CRRTO7W5WF54K53A3MRAFEUISHZ7Y5YGRVGRGHUZESLV5VYYWXI", + ToAddress: "GBLI2OE4H3HAW7Z2GXLYZQNQ57XLHJ5OILFPVL33EPA4GDAIQ5F33JGA", + SrcAssetCode: "USDC", + SrcAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5", + SrcAmount: 50, + DestAssetCode: "USDC", + DestAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5", + DestAmount: 50, + CreatedAt: time.Date(2024, 5, 28, 11, 0, 0, 0, time.UTC), + Memo: &expectedMemo, + }, payment) +} diff --git a/internal/utils/ingestion_utils.go b/internal/utils/ingestion_utils.go new file mode 100644 index 0000000..9cf4d7a --- /dev/null +++ b/internal/utils/ingestion_utils.go @@ -0,0 +1,78 @@ +package utils + +import ( + "strconv" + + "github.com/stellar/go/ingest" + "github.com/stellar/go/toid" + "github.com/stellar/go/xdr" +) + +func OperationID(ledgerNumber, txNumber, opNumber int32) int64 { + return toid.New(ledgerNumber, txNumber, opNumber).ToInt64() +} + +func OperationResult(tx ingest.LedgerTransaction, opNumber int) *xdr.OperationResultTr { + results, _ := tx.Result.OperationResults() + tr := results[opNumber-1].MustTr() + return &tr +} + +func TransactionID(ledgerNumber, txNumber int32) int64 { + return toid.New(int32(ledgerNumber), int32(txNumber), 0).ToInt64() +} + +func TransactionHash(ledgerMeta xdr.LedgerCloseMeta, txNumber int) string { + return ledgerMeta.TransactionHash(txNumber - 1).HexString() +} + +func Memo(memo xdr.Memo, txHash string) *string { + memoType := memo.Type + switch memoType { + case xdr.MemoTypeMemoNone: + return nil + case xdr.MemoTypeMemoText: + if text, ok := memo.GetText(); ok { + return &text + } + case xdr.MemoTypeMemoId: + if id, ok := memo.GetId(); ok { + idStr := strconv.FormatUint(uint64(id), 10) + return &idStr + } + case xdr.MemoTypeMemoHash: + if hash, ok := memo.GetHash(); ok { + hashStr := hash.HexString() + return &hashStr + } + case xdr.MemoTypeMemoReturn: + if retHash, ok := memo.GetRetHash(); ok { + retHashStr := retHash.HexString() + return &retHashStr + } + default: + // TODO: track in Sentry + // sentry.CaptureException(fmt.Errorf("unknown memo type %q for transaction %s", memoType.String(), txHash)) + return nil + } + + // TODO: track in Sentry + // sentry.CaptureException(fmt.Errorf("failed to parse memo for type %q and transaction %s", memoType.String(), txHash)) + return nil +} + +func SourceAccount(op xdr.Operation, tx ingest.LedgerTransaction) string { + account := op.SourceAccount + if account != nil { + return account.ToAccountId().Address() + } + + return tx.Envelope.SourceAccount().ToAccountId().Address() +} + +func AssetCode(asset xdr.Asset) string { + if asset.Type == xdr.AssetTypeAssetTypeNative { + return "XLM" + } + return SanitizeUTF8(asset.GetCode()) +} diff --git a/internal/utils/ingestion_utils_test.go b/internal/utils/ingestion_utils_test.go new file mode 100644 index 0000000..701b9b9 --- /dev/null +++ b/internal/utils/ingestion_utils_test.go @@ -0,0 +1,68 @@ +package utils + +import ( + "crypto/sha256" + "testing" + + "github.com/stellar/go/xdr" + "github.com/stretchr/testify/assert" +) + +func TestMemo(t *testing.T) { + t.Run("type_none", func(t *testing.T) { + memo := xdr.Memo{ + Type: xdr.MemoTypeMemoNone, + } + + result := Memo(memo, "") + assert.Equal(t, (*string)(nil), result) + }) + + t.Run("type_text", func(t *testing.T) { + value := "test" + memo := xdr.Memo{ + Type: xdr.MemoTypeMemoText, + Text: &value, + } + + result := Memo(memo, "") + assert.Equal(t, "test", *result) + }) + + t.Run("type_id", func(t *testing.T) { + value := xdr.Uint64(12345) + memo := xdr.Memo{ + Type: xdr.MemoTypeMemoId, + Id: &value, + } + + result := Memo(memo, "") + assert.Equal(t, "12345", *result) + }) + + t.Run("type_hash", func(t *testing.T) { + hash := sha256.New() + hash.Write([]byte("test")) + value := xdr.Hash(hash.Sum(nil)) + memo := xdr.Memo{ + Type: xdr.MemoTypeMemoHash, + Hash: &value, + } + + result := Memo(memo, "") + assert.Equal(t, value.HexString(), *result) + }) + + t.Run("type_return", func(t *testing.T) { + hash := sha256.New() + hash.Write([]byte("test")) + value := xdr.Hash(hash.Sum(nil)) + memo := xdr.Memo{ + Type: xdr.MemoTypeMemoReturn, + RetHash: &value, + } + + result := Memo(memo, "") + assert.Equal(t, value.HexString(), *result) + }) +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 3dbdc80..2a2ae42 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -1,6 +1,18 @@ package utils -import "reflect" +import ( + "bytes" + "reflect" + "strings" +) + +// SanitizeUTF8 sanitizes a string to comply to the UTF-8 character set and Postgres' code zero byte constraint +func SanitizeUTF8(input string) string { + // Postgres does not allow code zero bytes on the "text" type and will throw "invalid byte sequence" when encountering one + // https://www.postgresql.org/docs/13/datatype-character.html + bs := bytes.ReplaceAll([]byte(input), []byte{0}, []byte{}) + return strings.ToValidUTF8(string(bs), "?") +} // IsEmpty checks if a value is empty. func IsEmpty[T any](v T) bool { diff --git a/internal/utils/utils_test.go b/internal/utils/utils_test.go index 557d025..8c75b79 100644 --- a/internal/utils/utils_test.go +++ b/internal/utils/utils_test.go @@ -6,6 +6,18 @@ import ( "github.com/stretchr/testify/assert" ) +func TestSanitizeUTF8(t *testing.T) { + t.Run("non_utf8", func(t *testing.T) { + result := SanitizeUTF8("test\xF5") + assert.Equal(t, "test?", result) + }) + + t.Run("zero_byte", func(t *testing.T) { + result := SanitizeUTF8("test\x00\x00") + assert.Equal(t, "test", result) + }) +} + func TestUnwrapInterfaceToPointer(t *testing.T) { // Test with a string strValue := "test"