From a69ca69eeb320d47359160fb361771ed5369f135 Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Wed, 22 May 2024 14:38:37 -0300 Subject: [PATCH 01/14] Merge main into ingest --- cmd/root.go | 1 + go.mod | 41 ++++++++++++++ go.sum | 115 ++++++++++++++++++++++++++++++++++++++ internal/data/payments.go | 93 ++++++++++++++++++++++++++++++ internal/db/db.go | 2 + 5 files changed, 252 insertions(+) diff --git a/cmd/root.go b/cmd/root.go index 7a10ffc..0fc44f7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -34,4 +34,5 @@ func init() { logger.SetLevel(logrus.TraceLevel) rootCmd.AddCommand((&serveCmd{Logger: logger}).Command()) + rootCmd.AddCommand((&ingestCmd{Logger: logger}).Command()) } diff --git a/go.mod b/go.mod index eeb5532..f06d1f4 100644 --- a/go.mod +++ b/go.mod @@ -14,24 +14,44 @@ require ( ) require ( + cloud.google.com/go v0.112.0 // indirect + cloud.google.com/go/compute v1.23.3 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.5 // indirect + cloud.google.com/go/storage v1.37.0 // indirect github.com/BurntSushi/toml v1.3.2 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go v1.45.26 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/djherbis/fscache v0.10.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.5.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/schema v1.2.0 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -51,11 +71,32 @@ require ( github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.18.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/mod v0.13.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.14.0 // indirect + google.golang.org/api v0.157.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect + google.golang.org/grpc v1.60.1 // indirect google.golang.org/protobuf v1.32.0 // indirect + gopkg.in/djherbis/atime.v1 v1.0.0 // indirect + gopkg.in/djherbis/stream.v1 v1.3.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/tylerb/graceful.v1 v1.2.15 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 9cff354..db21dc2 100644 --- a/go.sum +++ b/go.sum @@ -17,14 +17,22 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -35,6 +43,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.37.0 h1:WI8CsaFO8Q9KjPVtsZ5Cmi0dXV25zMoX0FklT7c3Jm4= +cloud.google.com/go/storage v1.37.0/go.mod h1:i34TiT2IhiNDmcj65PqwCjcoUX7Z5pLzS8DEmoiFq1k= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= @@ -42,14 +52,20 @@ github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f h1:zvClvFQwU++UpIUBGC8YmDlfhUrweEy1R1Fj1gu5iIM= github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.45.26 h1:PJ2NJNY5N/yeobLYe1Y+xLdavBi67ZI8gvph6ftwVCg= +github.com/aws/aws-sdk-go v1.45.26/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -60,19 +76,27 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/djherbis/fscache v0.10.1 h1:hDv+RGyvD+UDKyRYuLoVNbuRTnf2SrA2K3VyR1br9lk= +github.com/djherbis/fscache v0.10.1/go.mod h1:yyPYtkNnnPXsW+81lAcQS6yab3G2CRfnPLotBvtbf0c= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -88,12 +112,19 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -116,6 +147,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -128,15 +160,19 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v0.0.0-20160401233042-9235644dd9e5 h1:oERTZ1buOUYlpmKaqlO5fYmz8cZ1rYu5DieJzF4ZVmU= github.com/google/go-querystring v0.0.0-20160401233042-9235644dd9e5/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -148,9 +184,17 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/schema v1.2.0 h1:YufUaxZYCKGFuAq3c96BOhjgd5nmXiOY9NGzF247Tsc= github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= @@ -158,6 +202,8 @@ github.com/guregu/null v4.0.0+incompatible h1:4zw0ckM7ECd6FNNddc3Fu4aty9nTlpkkzH github.com/guregu/null v4.0.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -168,6 +214,10 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jarcoal/httpmock v0.0.0-20161210151336-4442edb3db31 h1:Aw95BEvxJ3K6o9GGv5ppCd1P8hkeIeEJ30FO+OhOJpM= github.com/jarcoal/httpmock v0.0.0-20161210151336-4442edb3db31/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -207,6 +257,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -271,6 +323,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -298,12 +351,27 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -312,7 +380,10 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -348,6 +419,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -376,10 +450,13 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -391,6 +468,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -401,6 +480,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -436,11 +518,16 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -449,11 +536,15 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -501,10 +592,15 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -524,6 +620,8 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.157.0 h1:ORAeqmbrrozeyw5NjnMxh7peHO0UzV4wWYSwZeCUb20= +google.golang.org/api v0.157.0/go.mod h1:+z4v4ufbZ1WEpld6yMGHyggs+PmAHiaLNj5ytP3N01g= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -531,6 +629,8 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -567,6 +667,12 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= +google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457 h1:KHBtwE+eQc3+NxpjmRFlQ3pJQ2FNnhhgB9xOV8kyBuU= +google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -583,6 +689,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -601,6 +709,10 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/djherbis/atime.v1 v1.0.0 h1:eMRqB/JrLKocla2PBPKgQYg/p5UG4L6AUAs92aP7F60= +gopkg.in/djherbis/atime.v1 v1.0.0/go.mod h1:hQIUStKmJfvf7xdh/wtK84qe+DsTV5LnA9lzxxtPpJ8= +gopkg.in/djherbis/stream.v1 v1.3.1 h1:uGfmsOY1qqMjQQphhRBSGLyA9qumJ56exkRu9ASTjCw= +gopkg.in/djherbis/stream.v1 v1.3.1/go.mod h1:aEV8CBVRmSpLamVJfM903Npic1IKmb2qS30VAZ+sssg= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/gavv/httpexpect.v1 v1.0.0-20170111145843-40724cf1e4a0 h1:r5ptJ1tBxVAeqw4CrYWhXIMr0SybY3CDHuIbCg5CFVw= gopkg.in/gavv/httpexpect.v1 v1.0.0-20170111145843-40724cf1e4a0/go.mod h1:WtiW9ZA1LdaWqtQRo1VbIL/v4XZ8NDta+O/kSpGgVek= @@ -611,6 +723,9 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/tylerb/graceful.v1 v1.2.15 h1:1JmOyhKqAyX3BgTXMI84LwT6FOJ4tP2N9e2kwTCM0nQ= gopkg.in/tylerb/graceful.v1 v1.2.15/go.mod h1:yBhekWvR20ACXVObSSdD3u6S9DeSylanL2PAbAC/uJ8= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/data/payments.go b/internal/data/payments.go index 4c19ff1..d333a1d 100644 --- a/internal/data/payments.go +++ b/internal/data/payments.go @@ -2,15 +2,108 @@ package data import ( "context" + "database/sql" "fmt" + "time" "github.com/stellar/wallet-backend/internal/db" + + "github.com/jmoiron/sqlx" + "github.com/stellar/go/support/errors" ) type PaymentModel struct { db db.ConnectionPool } +type Payment struct { + OperationID int64 + OperationType string + TransactionID int64 + TransactionHash string + From string + To string + SrcAssetCode string + SrcAssetIssuer string + SrcAmount int64 + DestAssetCode string + DestAssetIssuer string + DestAmount int64 + CreatedAt time.Time + Memo string +} + +func (m *PaymentModel) GetLatestLedgerSynced(ctx context.Context, cursorName string) (uint32, error) { + var lastSyncedLedger uint32 + err := m.db.QueryRowxContext(ctx, `SELECT value FROM ingestion_store WHERE key = $1`, cursorName).Scan(&lastSyncedLedger) + // First run, key does not exist yet + if err == sql.ErrNoRows { + return 0, nil + } + if err != nil { + return 0, err + } + + return lastSyncedLedger, nil +} + +func (m *PaymentModel) UpdateLatestLedgerSynced(ctx context.Context, cursorName string, ledger uint32) error { + const query = ` + INSERT INTO ingestion_store (key, value) VALUES ($1, $2) + ON CONFLICT (key) DO UPDATE SET value = excluded.value + ` + _, err := m.db.ExecContext(ctx, query, cursorName, ledger) + if err != nil { + return errors.Wrapf(err, "updating last synced ledger to %d", ledger) + } + + return nil +} + +func (m *PaymentModel) BeginTx(ctx context.Context) (*sqlx.Tx, error) { + tx, err := m.db.BeginTxx(ctx, nil) + if err != nil { + return nil, errors.Wrap(err, "beginning transaction") + } + + return tx, nil +} + +func (m *PaymentModel) AddPayment(ctx context.Context, tx *sqlx.Tx, payment Payment) error { + const query = ` + INSERT INTO ingestion_payments ( + operation_id, operation_type, transaction_id, transaction_hash, from_address, to_address, src_asset_code, src_asset_issuer, src_amount, + dest_asset_code, dest_asset_issuer, dest_amount, created_at, memo + ) + SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14 + WHERE EXISTS ( + SELECT 1 FROM accounts WHERE stellar_address IN ($5, $6) + ) + ON CONFLICT (operation_id) DO UPDATE SET + operation_type = EXCLUDED.operation_type, + transaction_id = EXCLUDED.transaction_id, + transaction_hash = EXCLUDED.transaction_hash, + from_address = EXCLUDED.from_address, + to_address = EXCLUDED.to_address, + src_asset_code = EXCLUDED.src_asset_code, + src_asset_issuer = EXCLUDED.src_asset_issuer, + src_amount = EXCLUDED.src_amount, + dest_asset_code = EXCLUDED.dest_asset_code, + dest_asset_issuer = EXCLUDED.dest_asset_issuer, + dest_amount = EXCLUDED.dest_amount, + created_at = EXCLUDED.created_at, + memo = EXCLUDED.memo + ; + ` + _, err := tx.ExecContext(ctx, query, payment.OperationID, payment.OperationType, payment.TransactionID, payment.TransactionHash, payment.From, payment.To, payment.SrcAssetCode, payment.SrcAssetIssuer, payment.SrcAmount, + payment.DestAssetCode, payment.DestAssetIssuer, payment.DestAmount, payment.CreatedAt, payment.Memo) + if err != nil { + return errors.Wrap(err, "inserting payment") + } + + return nil +} + func (m *PaymentModel) SubscribeAddress(ctx context.Context, address string) error { const query = `INSERT INTO accounts (stellar_address) VALUES ($1) ON CONFLICT DO NOTHING` _, err := m.db.ExecContext(ctx, query, address) diff --git a/internal/db/db.go b/internal/db/db.go index 87fb273..ca7773b 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -2,6 +2,7 @@ package db import ( "context" + "database/sql" "fmt" "time" @@ -17,6 +18,7 @@ type ConnectionPool interface { sqlx.PreparerContext GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error + BeginTxx(ctx context.Context, opts *sql.TxOptions) (*sqlx.Tx, error) } // Make sure *DBConnectionPoolImplementation implements DBConnectionPool: From fb6fe067b51877bac7207380af94f975971b4e3c Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Wed, 22 May 2024 15:57:34 -0300 Subject: [PATCH 02/14] ingest: initial commit --- .gitignore | 3 +- cmd/ingest.go | 101 ++++++++ cmd/root.go | 12 +- cmd/serve.go | 16 +- internal/data/payments.go | 6 +- .../2024-05-22.0-ingest_payments.sql | 34 +++ .../ingest/config/stellar-core_pubnet.cfg | 25 ++ .../ingest/config/stellar-core_testnet.cfg | 25 ++ internal/ingest/ingest.go | 102 ++++++++ internal/serve/serve.go | 11 +- internal/services/ingest.go | 231 ++++++++++++++++++ internal/utils/ingestion_utils.go | 63 +++++ internal/utils/utils.go | 14 ++ 13 files changed, 615 insertions(+), 28 deletions(-) create mode 100644 cmd/ingest.go create mode 100644 internal/db/migrations/2024-05-22.0-ingest_payments.sql create mode 100644 internal/ingest/config/stellar-core_pubnet.cfg create mode 100644 internal/ingest/config/stellar-core_testnet.cfg create mode 100644 internal/ingest/ingest.go create mode 100644 internal/services/ingest.go create mode 100644 internal/utils/ingestion_utils.go create mode 100644 internal/utils/utils.go diff --git a/.gitignore b/.gitignore index 56f1c98..bb390c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ # Visual Studio Code -.vscode \ No newline at end of file +.vscode +captive-core*/ \ No newline at end of file diff --git a/cmd/ingest.go b/cmd/ingest.go new file mode 100644 index 0000000..73dbe76 --- /dev/null +++ b/cmd/ingest.go @@ -0,0 +1,101 @@ +package cmd + +import ( + "go/types" + + _ "github.com/lib/pq" + "github.com/spf13/cobra" + "github.com/stellar/go/network" + "github.com/stellar/go/support/config" + "github.com/stellar/go/support/log" + "github.com/stellar/wallet-backend/internal/ingest" +) + +type ingestCmd struct{} + +func (c *ingestCmd) Command() *cobra.Command { + cfg := ingest.Configs{} + cfgOpts := config.ConfigOptions{ + { + Name: "database-url", + Usage: "Database connection URL.", + OptType: types.String, + ConfigKey: &cfg.DatabaseURL, + FlagDefault: "postgres://postgres@localhost:5432/wallet-backend?sslmode=disable", + Required: true, + }, + { + Name: "network-passphrase", + Usage: "Stellar Network Passphrase to connect.", + OptType: types.String, + ConfigKey: &cfg.NetworkPassphrase, + FlagDefault: network.TestNetworkPassphrase, + Required: true, + }, + { + Name: "captive-core-bin-path", + Usage: "Path to Captive Core's binary file.", + OptType: types.String, + ConfigKey: &cfg.CaptiveCoreBinPath, + FlagDefault: "/usr/local/bin/stellar-core", + Required: true, + }, + { + Name: "captive-core-config-dir", + Usage: "Path to Captive Core's configuration files directory.", + OptType: types.String, + ConfigKey: &cfg.CaptiveCoreConfigDir, + FlagDefault: "./internal/ingest/config", + Required: true, + }, + { + Name: "ledger-cursor-name", + Usage: "Name of last synced ledger cursor. Attention: there should never be more than one container running with a same cursor name.", + OptType: types.String, + ConfigKey: &cfg.LedgerCursorName, + FlagDefault: "last_synced_ledger", + Required: false, + }, + { + Name: "start", + Usage: "Ledger number from which ingestion should start. When not present, ingestion will resume from last synced ledger.", + OptType: types.Int, + ConfigKey: &cfg.StartLedger, + FlagDefault: 0, + Required: false, + }, + { + Name: "end", + Usage: "Ledger number up to which ingestion should run. When not present, ingestion run indefinitely (live ingestion requires it to be empty).", + OptType: types.Int, + ConfigKey: &cfg.EndLedger, + FlagDefault: 0, + Required: false, + }, + } + + cmd := &cobra.Command{ + Use: "ingest", + Short: "Run Ingestion service", + PersistentPreRun: func(_ *cobra.Command, _ []string) { + cfgOpts.Require() + if err := cfgOpts.SetValues(); err != nil { + log.Fatalf("Error setting values of config options: %s", err.Error()) + } + }, + Run: func(_ *cobra.Command, _ []string) { + c.Run(cfg) + }, + } + if err := cfgOpts.Init(cmd); err != nil { + log.Fatalf("Error initializing a config option: %s", err.Error()) + } + return cmd +} + +func (c *ingestCmd) Run(cfg ingest.Configs) { + err := ingest.Ingest(cfg) + if err != nil { + log.Fatalf("Error running Ingest: %s", err.Error()) + } +} diff --git a/cmd/root.go b/cmd/root.go index 0fc44f7..3fc1ade 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,11 +1,9 @@ package cmd import ( - "log" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" - supportlog "github.com/stellar/go/support/log" + "github.com/stellar/go/support/log" ) // rootCmd represents the base command when called without any subcommands @@ -30,9 +28,9 @@ func Execute() { } func init() { - logger := supportlog.New() - logger.SetLevel(logrus.TraceLevel) + log.DefaultLogger = log.New() + log.DefaultLogger.SetLevel(logrus.TraceLevel) - rootCmd.AddCommand((&serveCmd{Logger: logger}).Command()) - rootCmd.AddCommand((&ingestCmd{Logger: logger}).Command()) + rootCmd.AddCommand((&serveCmd{}).Command()) + rootCmd.AddCommand((&ingestCmd{}).Command()) } diff --git a/cmd/serve.go b/cmd/serve.go index 0211c46..6398cfa 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -6,18 +6,14 @@ import ( _ "github.com/lib/pq" "github.com/spf13/cobra" "github.com/stellar/go/support/config" - supportlog "github.com/stellar/go/support/log" + "github.com/stellar/go/support/log" "github.com/stellar/wallet-backend/internal/serve" ) -type serveCmd struct { - Logger *supportlog.Entry -} +type serveCmd struct{} func (c *serveCmd) Command() *cobra.Command { - cfg := serve.Configs{ - Logger: c.Logger, - } + cfg := serve.Configs{} cfgOpts := config.ConfigOptions{ { Name: "port", @@ -42,13 +38,13 @@ func (c *serveCmd) Command() *cobra.Command { Run: func(_ *cobra.Command, _ []string) { cfgOpts.Require() if err := cfgOpts.SetValues(); err != nil { - c.Logger.Fatalf("Error setting values of config options: %s", err.Error()) + log.Fatalf("Error setting values of config options: %s", err.Error()) } c.Run(cfg) }, } if err := cfgOpts.Init(cmd); err != nil { - c.Logger.Fatalf("Error initializing a config option: %s", err.Error()) + log.Fatalf("Error initializing a config option: %s", err.Error()) } return cmd } @@ -56,6 +52,6 @@ func (c *serveCmd) Command() *cobra.Command { func (c *serveCmd) Run(cfg serve.Configs) { err := serve.Serve(cfg) if err != nil { - c.Logger.Fatalf("Error running Serve: %s", err.Error()) + log.Fatalf("Error running Serve: %s", err.Error()) } } diff --git a/internal/data/payments.go b/internal/data/payments.go index d333a1d..06f0d8e 100644 --- a/internal/data/payments.go +++ b/internal/data/payments.go @@ -35,7 +35,7 @@ type Payment struct { func (m *PaymentModel) GetLatestLedgerSynced(ctx context.Context, cursorName string) (uint32, error) { var lastSyncedLedger uint32 - err := m.db.QueryRowxContext(ctx, `SELECT value FROM ingestion_store WHERE key = $1`, cursorName).Scan(&lastSyncedLedger) + err := m.db.QueryRowxContext(ctx, `SELECT value FROM ingest_store WHERE key = $1`, cursorName).Scan(&lastSyncedLedger) // First run, key does not exist yet if err == sql.ErrNoRows { return 0, nil @@ -49,7 +49,7 @@ func (m *PaymentModel) GetLatestLedgerSynced(ctx context.Context, cursorName str func (m *PaymentModel) UpdateLatestLedgerSynced(ctx context.Context, cursorName string, ledger uint32) error { const query = ` - INSERT INTO ingestion_store (key, value) VALUES ($1, $2) + INSERT INTO ingest_store (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = excluded.value ` _, err := m.db.ExecContext(ctx, query, cursorName, ledger) @@ -71,7 +71,7 @@ func (m *PaymentModel) BeginTx(ctx context.Context) (*sqlx.Tx, error) { func (m *PaymentModel) AddPayment(ctx context.Context, tx *sqlx.Tx, payment Payment) error { const query = ` - INSERT INTO ingestion_payments ( + INSERT INTO ingest_payments ( operation_id, operation_type, transaction_id, transaction_hash, from_address, to_address, src_asset_code, src_asset_issuer, src_amount, dest_asset_code, dest_asset_issuer, dest_amount, created_at, memo ) diff --git a/internal/db/migrations/2024-05-22.0-ingest_payments.sql b/internal/db/migrations/2024-05-22.0-ingest_payments.sql new file mode 100644 index 0000000..3f4bea4 --- /dev/null +++ b/internal/db/migrations/2024-05-22.0-ingest_payments.sql @@ -0,0 +1,34 @@ +-- +migrate Up + +CREATE TABLE ingest_store ( + key varchar(255) NOT NULL, + value varchar(255) NOT NULL, + PRIMARY KEY (key) +); + +CREATE TABLE ingest_payments ( + operation_id bigint NOT NULL, + operation_type text NOT NULL, + transaction_id bigint NOT NULL, + transaction_hash text NOT NULL, + from_address text NOT NULL, + to_address text NOT NULL, + src_asset_code text NOT NULL, + src_asset_issuer text NOT NULL, + src_amount bigint NOT NULL, + dest_asset_code text NOT NULL, + dest_asset_issuer text NOT NULL, + dest_amount bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + memo text, + PRIMARY KEY (operation_id) +); + +CREATE INDEX from_address_idx ON ingest_payments (from_address); +CREATE INDEX to_address_idx ON ingest_payments (to_address); + +-- +migrate Down + +DROP TABLE ingest_payments; + +DROP TABLE ingest_store; diff --git a/internal/ingest/config/stellar-core_pubnet.cfg b/internal/ingest/config/stellar-core_pubnet.cfg new file mode 100644 index 0000000..8394e87 --- /dev/null +++ b/internal/ingest/config/stellar-core_pubnet.cfg @@ -0,0 +1,25 @@ +# Stellar Pubnet validators +[[HOME_DOMAINS]] +HOME_DOMAIN="www.stellar.org" +QUALITY="HIGH" + +[[VALIDATORS]] +NAME="SDF 1" +PUBLIC_KEY="GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH" +ADDRESS="core-live-a.stellar.org:11625" +HISTORY="curl -sf http://history.stellar.org/prd/core-live/core_live_001/{0} -o {1}" +HOME_DOMAIN="www.stellar.org" + +[[VALIDATORS]] +NAME="SDF 2" +PUBLIC_KEY="GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK" +ADDRESS="core-live-b.stellar.org:11625" +HISTORY="curl -sf http://history.stellar.org/prd/core-live/core_live_002/{0} -o {1}" +HOME_DOMAIN="www.stellar.org" + +[[VALIDATORS]] +NAME="SDF 3" +PUBLIC_KEY="GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ" +ADDRESS="core-live-c.stellar.org:11625" +HISTORY="curl -sf http://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" +HOME_DOMAIN="www.stellar.org" \ No newline at end of file diff --git a/internal/ingest/config/stellar-core_testnet.cfg b/internal/ingest/config/stellar-core_testnet.cfg new file mode 100644 index 0000000..357470a --- /dev/null +++ b/internal/ingest/config/stellar-core_testnet.cfg @@ -0,0 +1,25 @@ +# Stellar Testnet validators +[[HOME_DOMAINS]] +HOME_DOMAIN="testnet.stellar.org" +QUALITY="HIGH" + +[[VALIDATORS]] +NAME="sdftest1" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" +ADDRESS="core-testnet1.stellar.org" +HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_001/{0} -o {1}" + +[[VALIDATORS]] +NAME="sdftest2" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GCUCJTIYXSOXKBSNFGNFWW5MUQ54HKRPGJUTQFJ5RQXZXNOLNXYDHRAP" +ADDRESS="core-testnet2.stellar.org" +HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_002/{0} -o {1}" + +[[VALIDATORS]] +NAME="sdftest3" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GC2V2EFSXN6SQTWVYA5EPJPBWWIMSD2XQNKUOHGEKB535AQE2I6IXV2Z" +ADDRESS="core-testnet3.stellar.org" +HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_003/{0} -o {1}" \ No newline at end of file diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go new file mode 100644 index 0000000..e94a601 --- /dev/null +++ b/internal/ingest/ingest.go @@ -0,0 +1,102 @@ +package ingest + +import ( + "context" + "fmt" + + "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/network" + "github.com/stellar/go/support/log" + "github.com/stellar/wallet-backend/internal/data" + "github.com/stellar/wallet-backend/internal/db" + "github.com/stellar/wallet-backend/internal/services" +) + +type Configs struct { + DatabaseURL string + NetworkPassphrase string + CaptiveCoreBinPath string + CaptiveCoreConfigDir string + LedgerCursorName string + StartLedger int + EndLedger int +} + +func Ingest(cfg Configs) error { + ctx := context.Background() + + manager, err := setupDeps(cfg) + if err != nil { + log.Ctx(ctx).Fatalf("Error setting up dependencies for ingest: %v", err) + } + + if err = manager.Run(ctx, uint32(cfg.StartLedger), uint32(cfg.EndLedger)); err != nil { + log.Ctx(ctx).Fatalf("Running ingest from %d to %d: %v", cfg.StartLedger, cfg.EndLedger, err) + } + + return nil +} + +func setupDeps(cfg Configs) (*services.IngestManager, error) { + // Open DB connection pool + dbConnectionPool, err := db.OpenDBConnectionPool(cfg.DatabaseURL) + if err != nil { + return nil, fmt.Errorf("error connecting to the database: %w", err) + } + models, err := data.NewModels(dbConnectionPool) + if err != nil { + return nil, fmt.Errorf("error creating models for Serve: %w", err) + } + + // Setup Captive Core backend + captiveCoreConfig, err := getCaptiveCoreConfig(cfg) + if err != nil { + return nil, fmt.Errorf("getting captive core config: %w", err) + } + ledgerBackend, err := ledgerbackend.NewCaptive(captiveCoreConfig) + if err != nil { + return nil, fmt.Errorf("creating captive core backend: %w", err) + } + + return &services.IngestManager{ + NetworkPassphrase: cfg.NetworkPassphrase, + LedgerCursorName: cfg.LedgerCursorName, + LedgerBackend: ledgerBackend, + PaymentModel: models.Payments, + }, nil +} + +func getCaptiveCoreConfig(cfg Configs) (ledgerbackend.CaptiveCoreConfig, error) { + var networkArchivesURLs []string + var configFilePath string + + switch cfg.NetworkPassphrase { + case network.TestNetworkPassphrase: + networkArchivesURLs = network.TestNetworkhistoryArchiveURLs + configFilePath = cfg.CaptiveCoreConfigDir + "/stellar-core_testnet.cfg" + case network.PublicNetworkPassphrase: + networkArchivesURLs = network.PublicNetworkhistoryArchiveURLs + configFilePath = cfg.CaptiveCoreConfigDir + "/stellar-core_pubnet.cfg" + default: + return ledgerbackend.CaptiveCoreConfig{}, fmt.Errorf("unknown network: %s", cfg.NetworkPassphrase) + } + + // Read configuration TOML + captiveCoreToml, err := ledgerbackend.NewCaptiveCoreTomlFromFile(configFilePath, ledgerbackend.CaptiveCoreTomlParams{ + CoreBinaryPath: cfg.CaptiveCoreBinPath, + NetworkPassphrase: cfg.NetworkPassphrase, + HistoryArchiveURLs: networkArchivesURLs, + UseDB: true, + }) + if err != nil { + return ledgerbackend.CaptiveCoreConfig{}, fmt.Errorf("creating captive core toml: %w", err) + } + + return ledgerbackend.CaptiveCoreConfig{ + NetworkPassphrase: cfg.NetworkPassphrase, + HistoryArchiveURLs: networkArchivesURLs, + BinaryPath: cfg.CaptiveCoreBinPath, + Toml: captiveCoreToml, + UseDB: true, + }, nil +} diff --git a/internal/serve/serve.go b/internal/serve/serve.go index 562ad84..e7c7be3 100644 --- a/internal/serve/serve.go +++ b/internal/serve/serve.go @@ -6,7 +6,7 @@ import ( "github.com/go-chi/chi" supporthttp "github.com/stellar/go/support/http" - supportlog "github.com/stellar/go/support/log" + "github.com/stellar/go/support/log" "github.com/stellar/go/support/render/health" "github.com/stellar/wallet-backend/internal/data" "github.com/stellar/wallet-backend/internal/db" @@ -15,13 +15,11 @@ import ( ) type Configs struct { - Logger *supportlog.Entry Port int DatabaseURL string } type handlerDeps struct { - Logger *supportlog.Entry Models *data.Models } @@ -36,10 +34,10 @@ func Serve(cfg Configs) error { ListenAddr: addr, Handler: handler(deps), OnStarting: func() { - deps.Logger.Infof("Starting Wallet Backend server on %s", addr) + log.Infof("Starting Wallet Backend server on %s", addr) }, OnStopping: func() { - deps.Logger.Info("Stopping Wallet Backend server") + log.Info("Stopping Wallet Backend server") }, }) @@ -57,13 +55,12 @@ func getHandlerDeps(cfg Configs) (handlerDeps, error) { } return handlerDeps{ - Logger: cfg.Logger, Models: models, }, nil } func handler(deps handlerDeps) http.Handler { - mux := supporthttp.NewAPIMux(deps.Logger) + mux := supporthttp.NewAPIMux(log.DefaultLogger) mux.NotFound(httperror.ErrorHandler{Error: httperror.NotFound}.ServeHTTP) mux.MethodNotAllowed(httperror.ErrorHandler{Error: httperror.MethodNotAllowed}.ServeHTTP) diff --git a/internal/services/ingest.go b/internal/services/ingest.go new file mode 100644 index 0000000..b42be47 --- /dev/null +++ b/internal/services/ingest.go @@ -0,0 +1,231 @@ +package services + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/pkg/errors" + "github.com/stellar/go/ingest" + "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/support/log" + "github.com/stellar/go/xdr" + "github.com/stellar/wallet-backend/internal/data" + "github.com/stellar/wallet-backend/internal/utils" +) + +type IngestManager struct { + PaymentModel *data.PaymentModel + LedgerBackend ledgerbackend.LedgerBackend + NetworkPassphrase string + LedgerCursorName string +} + +func (m *IngestManager) Run(ctx context.Context, start, end uint32) error { + var ingestLedger uint32 + + if start == 0 { + lastSyncedLedger, err := m.PaymentModel.GetLatestLedgerSynced(ctx, m.LedgerCursorName) + if err != nil { + return fmt.Errorf("getting last ledger synced: %w", err) + } + + if lastSyncedLedger == 0 { + return errors.New("ingestion service is not initialized, --start flag is required") + } + ingestLedger = lastSyncedLedger + } else { + ingestLedger = start + } + + if end != 0 && ingestLedger > end { + return fmt.Errorf("starting ledger (%d) may not be greater than ending ledger (%d)", ingestLedger, end) + } + + err := m.maybePrepareRange(ctx, ingestLedger, end) + if err != nil { + return fmt.Errorf("preparing range from %d to %d: %w", ingestLedger, end, err) + } + + heartbeat := make(chan any) + go trackServiceHealth(heartbeat) + + for ; end == 0 || ingestLedger <= end; ingestLedger++ { + log.Ctx(ctx).Infof("waiting for ledger %d", ingestLedger) + + ledgerMeta, err := m.LedgerBackend.GetLedger(ctx, ingestLedger) + if err != nil { + return fmt.Errorf("getting ledger meta for ledger %d: %w", ingestLedger, err) + } + + heartbeat <- true + + err = m.processLedger(ctx, ingestLedger, ledgerMeta) + if err != nil { + return fmt.Errorf("processing ledger %d: %w", ingestLedger, err) + } + + log.Ctx(ctx).Infof("ledger %d successfully processed", ingestLedger) + } + + return nil +} + +func (m *IngestManager) maybePrepareRange(ctx context.Context, from, to uint32) error { + var ledgerRange ledgerbackend.Range + if to == 0 { + ledgerRange = ledgerbackend.UnboundedRange(from) + } else { + ledgerRange = ledgerbackend.BoundedRange(from, to) + } + + prepared, err := m.LedgerBackend.IsPrepared(ctx, ledgerRange) + if err != nil { + return fmt.Errorf("checking prepared range: %w", err) + } + + if !prepared { + err = m.LedgerBackend.PrepareRange(ctx, ledgerRange) + if err != nil { + return fmt.Errorf("preparing range: %w", err) + } + } + + return nil +} + +func trackServiceHealth(heartbeat chan any) { + const alertAfter = time.Second * 60 + ticker := time.NewTicker(alertAfter) + + for { + select { + case <-ticker.C: + warn := fmt.Sprintf("ingestion service stale for over %s", alertAfter) + log.Warn(warn) + // TODO: track in Sentry + // sentry.CaptureMessage(warn) + ticker.Reset(alertAfter) + case <-heartbeat: + ticker.Reset(alertAfter) + } + } +} + +func (m *IngestManager) processLedger(ctx context.Context, ledger uint32, ledgerMeta xdr.LedgerCloseMeta) (err error) { + reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(m.NetworkPassphrase, ledgerMeta) + if err != nil { + return errors.Wrap(err, "creating ledger reader") + } + + ledgerCloseTime := time.Unix(int64(ledgerMeta.LedgerHeaderHistoryEntry().Header.ScpValue.CloseTime), 0).UTC() + ledgerSequence := ledgerMeta.LedgerSequence() + + dbTx, err := m.PaymentModel.BeginTx(ctx) + if err != nil { + return err + } + defer func() { + if err != nil { + rollbackErr := dbTx.Rollback() + if rollbackErr != nil { + log.Ctx(ctx).Error(rollbackErr) + } + } + }() + + for { + tx, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + return errors.Wrap(err, "reading transaction") + } + + if !tx.Result.Successful() { + continue + } + + txHash := utils.TransactionHash(ledgerMeta, int(tx.Index)) + txMemo := utils.Memo(tx.Envelope.Memo(), txHash) + + for idx, op := range tx.Envelope.Operations() { + opIdx := idx + 1 + + payment := data.Payment{ + OperationID: utils.OperationID(int32(ledgerSequence), int32(tx.Index), int32(opIdx)), + OperationType: op.Body.Type.String(), + TransactionID: utils.TransactionID(int32(ledgerSequence), int32(tx.Index)), + TransactionHash: txHash, + From: utils.SourceAccount(op, tx), + CreatedAt: ledgerCloseTime, + Memo: utils.SanitizeUTF8(txMemo), // Field is subject to user input + } + + switch op.Body.Type { + case xdr.OperationTypePayment: + fillPayment(&payment, op.Body) + case xdr.OperationTypePathPaymentStrictSend: + fillPathSend(&payment, op.Body, tx, opIdx) + case xdr.OperationTypePathPaymentStrictReceive: + fillPathReceive(&payment, op.Body, tx, opIdx) + default: + continue + } + + err = m.PaymentModel.AddPayment(ctx, dbTx, payment) + if err != nil { + return errors.Wrapf(err, "adding payment for ledger %d, tx %q (%d), operation %d (%d)", ledgerSequence, txHash, tx.Index, payment.OperationID, opIdx) + } + } + } + + err = m.PaymentModel.UpdateLatestLedgerSynced(ctx, m.LedgerCursorName, ledger) + if err != nil { + return err + } + + err = dbTx.Commit() + if err != nil { + return err + } + + return nil +} + +func fillPayment(payment *data.Payment, operation xdr.OperationBody) { + paymentOp := operation.MustPaymentOp() + payment.To = paymentOp.Destination.Address() + payment.SrcAssetCode = utils.AssetCode(paymentOp.Asset) + payment.DestAssetCode = payment.SrcAssetCode + payment.SrcAssetIssuer = paymentOp.Asset.GetIssuer() + payment.DestAssetIssuer = payment.SrcAssetIssuer + payment.SrcAmount = int64(paymentOp.Amount) + payment.DestAmount = int64(paymentOp.Amount) +} + +func fillPathSend(payment *data.Payment, operation xdr.OperationBody, transaction ingest.LedgerTransaction, operationIdx int) { + pathOp := operation.MustPathPaymentStrictSendOp() + result := utils.OperationResult(transaction, operationIdx).MustPathPaymentStrictSendResult() + payment.To = pathOp.Destination.Address() + payment.SrcAssetCode = utils.AssetCode(pathOp.SendAsset) + payment.DestAssetCode = utils.AssetCode(pathOp.DestAsset) + payment.SrcAssetIssuer = pathOp.SendAsset.GetIssuer() + payment.DestAssetIssuer = pathOp.DestAsset.GetIssuer() + payment.SrcAmount = int64(pathOp.SendAmount) + payment.DestAmount = int64(result.DestAmount()) +} + +func fillPathReceive(payment *data.Payment, operation xdr.OperationBody, transaction ingest.LedgerTransaction, operationIdx int) { + pathOp := operation.MustPathPaymentStrictReceiveOp() + result := utils.OperationResult(transaction, operationIdx).MustPathPaymentStrictReceiveResult() + payment.To = pathOp.Destination.Address() + payment.SrcAssetCode = utils.AssetCode(pathOp.SendAsset) + payment.DestAssetCode = utils.AssetCode(pathOp.DestAsset) + payment.SrcAssetIssuer = pathOp.SendAsset.GetIssuer() + payment.DestAssetIssuer = pathOp.DestAsset.GetIssuer() + payment.SrcAmount = int64(result.SendAmount()) + payment.DestAmount = int64(pathOp.DestAmount) +} diff --git a/internal/utils/ingestion_utils.go b/internal/utils/ingestion_utils.go new file mode 100644 index 0000000..efebad3 --- /dev/null +++ b/internal/utils/ingestion_utils.go @@ -0,0 +1,63 @@ +package utils + +import ( + "strconv" + + "github.com/stellar/go/ingest" + "github.com/stellar/go/toid" + "github.com/stellar/go/xdr" +) + +func OperationID(ledgerNumber, txNumber, opNumber int32) int64 { + return toid.New(ledgerNumber, txNumber, opNumber).ToInt64() +} + +func OperationResult(transaction ingest.LedgerTransaction, opNumber int) *xdr.OperationResultTr { + results, _ := transaction.Result.OperationResults() + tr := results[opNumber-1].MustTr() + return &tr +} + +func TransactionID(ledgerNumber, txNumber int32) int64 { + return toid.New(int32(ledgerNumber), int32(txNumber), 0).ToInt64() +} + +func TransactionHash(ledgerMeta xdr.LedgerCloseMeta, txNumber int) string { + return ledgerMeta.TransactionHash(txNumber - 1).HexString() +} + +func Memo(memo xdr.Memo, txHash string) string { + memoType := memo.Type + switch memoType { + case xdr.MemoTypeMemoNone: + return "" + case xdr.MemoTypeMemoText: + return memo.MustText() + case xdr.MemoTypeMemoId: + return strconv.FormatUint(uint64(memo.MustId()), 10) + case xdr.MemoTypeMemoHash: + return memo.MustHash().HexString() + case xdr.MemoTypeMemoReturn: + return memo.MustRetHash().HexString() + default: + // TODO: track in Sentry + // sentry.CaptureException(fmt.Errorf("unknown memo type %q for transaction %s", memoType.String(), txHash)) + return "" + } +} + +func SourceAccount(op xdr.Operation, tx ingest.LedgerTransaction) string { + account := op.SourceAccount + if account != nil { + return account.ToAccountId().Address() + } + + return tx.Envelope.SourceAccount().ToAccountId().Address() +} + +func AssetCode(asset xdr.Asset) string { + if asset.Type == xdr.AssetTypeAssetTypeNative { + return "XLM" + } + return SanitizeUTF8(asset.GetCode()) +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go new file mode 100644 index 0000000..979e32a --- /dev/null +++ b/internal/utils/utils.go @@ -0,0 +1,14 @@ +package utils + +import ( + "bytes" + "strings" +) + +// SanitizeUTF8 sanitizes a string to comply to the UTF-8 character set and Postgres' code zero byte constraint +func SanitizeUTF8(input string) string { + // Postgres does not allow code zero bytes on the "text" type and will throw "invalid byte sequence" when encountering one + // https://www.postgresql.org/docs/13/datatype-character.html + bs := bytes.ReplaceAll([]byte(input), []byte{0}, []byte{}) + return strings.ToValidUTF8(string(bs), "?") +} From df0835877d64fec63a892a6a0f61afb9a146106f Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Wed, 22 May 2024 16:06:22 -0300 Subject: [PATCH 03/14] tests --- internal/data/payments_test.go | 93 ++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/internal/data/payments_test.go b/internal/data/payments_test.go index dd7a44b..03d78c3 100644 --- a/internal/data/payments_test.go +++ b/internal/data/payments_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "testing" + "time" "github.com/stellar/go/keypair" "github.com/stellar/wallet-backend/internal/db" @@ -12,6 +13,98 @@ import ( "github.com/stretchr/testify/require" ) +func TestAddPayment(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + m := &PaymentModel{ + db: dbConnectionPool, + } + ctx := context.Background() + + const ( + fromAddress = "GCYQBVCREYSLKHHOWLT27VNZNGVIXXAPYVNNOWMQV67WVDD4PP2VZAX7" + toAddress = "GDDEAH46MNFO6JD7NTQ5FWJBC4ZSA47YEK3RKFHQWADYTS6NDVD5CORW" + ) + payment := Payment{ + OperationID: 2120562792996865, + OperationType: "OperationTypePayment", + TransactionID: 2120562792996864, + TransactionHash: "a3daffa64dc46db84888b1206dc8014a480042e7fe8b19fd5d05465709f4e887", + From: fromAddress, + To: toAddress, + SrcAssetCode: "USDC", + SrcAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5", + SrcAmount: 500000000, + DestAssetCode: "USDC", + DestAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5", + DestAmount: 500000000, + CreatedAt: time.Date(2023, 12, 15, 1, 0, 0, 0, time.UTC), + Memo: "", + } + + addPayment := func() { + tx, err := m.BeginTx(ctx) + require.NoError(t, err) + + err = m.AddPayment(ctx, tx, payment) + require.NoError(t, err) + + err = tx.Commit() + require.NoError(t, err) + } + + fetchPaymentInserted := func() bool { + var inserted bool + err := dbConnectionPool.QueryRowxContext(ctx, "SELECT EXISTS(SELECT 1 FROM ingest_payments)").Scan(&inserted) + require.NoError(t, err) + + return inserted + } + + cleanUpDB := func() { + _, err := dbConnectionPool.ExecContext(ctx, `DELETE FROM accounts; DELETE FROM ingest_payments;`) + require.NoError(t, err) + } + + t.Run("unkown_address", func(t *testing.T) { + addPayment() + + inserted := fetchPaymentInserted() + assert.False(t, inserted) + + cleanUpDB() + }) + + t.Run("from_known_address", func(t *testing.T) { + _, err := dbConnectionPool.ExecContext(ctx, `INSERT INTO accounts (stellar_address) VALUES ($1)`, fromAddress) + require.NoError(t, err) + + addPayment() + + inserted := fetchPaymentInserted() + assert.True(t, inserted) + + cleanUpDB() + }) + + t.Run("to_known_address", func(t *testing.T) { + _, err := dbConnectionPool.ExecContext(ctx, `INSERT INTO accounts (stellar_address) VALUES ($1)`, toAddress) + require.NoError(t, err) + + addPayment() + + inserted := fetchPaymentInserted() + assert.True(t, inserted) + + cleanUpDB() + }) +} + func TestSubscribeAddress(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() From 94ef4acd64b9d72a8a9ef8364cd8cd710b835bfb Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Wed, 22 May 2024 16:35:13 -0300 Subject: [PATCH 04/14] errors using fmt --- internal/data/payments.go | 7 +++---- internal/serve/serve.go | 2 +- internal/services/ingest.go | 8 ++++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/internal/data/payments.go b/internal/data/payments.go index 06f0d8e..d8a4395 100644 --- a/internal/data/payments.go +++ b/internal/data/payments.go @@ -9,7 +9,6 @@ import ( "github.com/stellar/wallet-backend/internal/db" "github.com/jmoiron/sqlx" - "github.com/stellar/go/support/errors" ) type PaymentModel struct { @@ -54,7 +53,7 @@ func (m *PaymentModel) UpdateLatestLedgerSynced(ctx context.Context, cursorName ` _, err := m.db.ExecContext(ctx, query, cursorName, ledger) if err != nil { - return errors.Wrapf(err, "updating last synced ledger to %d", ledger) + return fmt.Errorf("updating last synced ledger to %d: %w", ledger, err) } return nil @@ -63,7 +62,7 @@ func (m *PaymentModel) UpdateLatestLedgerSynced(ctx context.Context, cursorName func (m *PaymentModel) BeginTx(ctx context.Context) (*sqlx.Tx, error) { tx, err := m.db.BeginTxx(ctx, nil) if err != nil { - return nil, errors.Wrap(err, "beginning transaction") + return nil, fmt.Errorf("beginning transaction: %w", err) } return tx, nil @@ -98,7 +97,7 @@ func (m *PaymentModel) AddPayment(ctx context.Context, tx *sqlx.Tx, payment Paym _, err := tx.ExecContext(ctx, query, payment.OperationID, payment.OperationType, payment.TransactionID, payment.TransactionHash, payment.From, payment.To, payment.SrcAssetCode, payment.SrcAssetIssuer, payment.SrcAmount, payment.DestAssetCode, payment.DestAssetIssuer, payment.DestAmount, payment.CreatedAt, payment.Memo) if err != nil { - return errors.Wrap(err, "inserting payment") + return fmt.Errorf("inserting payment: %w", err) } return nil diff --git a/internal/serve/serve.go b/internal/serve/serve.go index e7c7be3..db3ffea 100644 --- a/internal/serve/serve.go +++ b/internal/serve/serve.go @@ -34,7 +34,7 @@ func Serve(cfg Configs) error { ListenAddr: addr, Handler: handler(deps), OnStarting: func() { - log.Infof("Starting Wallet Backend server on %s", addr) + log.Infof("Starting Wallet Backend server on port %d", cfg.Port) }, OnStopping: func() { log.Info("Stopping Wallet Backend server") diff --git a/internal/services/ingest.go b/internal/services/ingest.go index b42be47..50c9312 100644 --- a/internal/services/ingest.go +++ b/internal/services/ingest.go @@ -2,11 +2,11 @@ package services import ( "context" + "errors" "fmt" "io" "time" - "github.com/pkg/errors" "github.com/stellar/go/ingest" "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/support/log" @@ -116,7 +116,7 @@ func trackServiceHealth(heartbeat chan any) { func (m *IngestManager) processLedger(ctx context.Context, ledger uint32, ledgerMeta xdr.LedgerCloseMeta) (err error) { reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(m.NetworkPassphrase, ledgerMeta) if err != nil { - return errors.Wrap(err, "creating ledger reader") + return fmt.Errorf("creating ledger reader: %w", err) } ledgerCloseTime := time.Unix(int64(ledgerMeta.LedgerHeaderHistoryEntry().Header.ScpValue.CloseTime), 0).UTC() @@ -141,7 +141,7 @@ func (m *IngestManager) processLedger(ctx context.Context, ledger uint32, ledger break } if err != nil { - return errors.Wrap(err, "reading transaction") + return fmt.Errorf("reading transaction: %w", err) } if !tx.Result.Successful() { @@ -177,7 +177,7 @@ func (m *IngestManager) processLedger(ctx context.Context, ledger uint32, ledger err = m.PaymentModel.AddPayment(ctx, dbTx, payment) if err != nil { - return errors.Wrapf(err, "adding payment for ledger %d, tx %q (%d), operation %d (%d)", ledgerSequence, txHash, tx.Index, payment.OperationID, opIdx) + return fmt.Errorf("adding payment for ledger %d, tx %q (%d), operation %d (%d): %w", ledgerSequence, txHash, tx.Index, payment.OperationID, opIdx, err) } } } From c81a98518778990bedbd71a3f5afa01968b66715 Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Mon, 27 May 2024 10:58:49 -0300 Subject: [PATCH 05/14] rework: custom setters for path variables --- cmd/ingest.go | 27 +++++---- cmd/utils/custom_set_value.go | 62 ++++++++++++++++++- cmd/utils/custom_set_value_test.go | 96 ++++++++++++++++++++++++++++++ internal/ingest/ingest.go | 10 +++- 4 files changed, 180 insertions(+), 15 deletions(-) diff --git a/cmd/ingest.go b/cmd/ingest.go index 73dbe76..73bf81f 100644 --- a/cmd/ingest.go +++ b/cmd/ingest.go @@ -8,6 +8,7 @@ import ( "github.com/stellar/go/network" "github.com/stellar/go/support/config" "github.com/stellar/go/support/log" + "github.com/stellar/wallet-backend/cmd/utils" "github.com/stellar/wallet-backend/internal/ingest" ) @@ -33,20 +34,22 @@ func (c *ingestCmd) Command() *cobra.Command { Required: true, }, { - Name: "captive-core-bin-path", - Usage: "Path to Captive Core's binary file.", - OptType: types.String, - ConfigKey: &cfg.CaptiveCoreBinPath, - FlagDefault: "/usr/local/bin/stellar-core", - Required: true, + Name: "captive-core-bin-path", + Usage: "Path to Captive Core's binary file.", + OptType: types.String, + CustomSetValue: utils.SetConfigOptionCaptiveCoreBinPath, + ConfigKey: &cfg.CaptiveCoreBinPath, + FlagDefault: "/usr/local/bin/stellar-core", + Required: true, }, { - Name: "captive-core-config-dir", - Usage: "Path to Captive Core's configuration files directory.", - OptType: types.String, - ConfigKey: &cfg.CaptiveCoreConfigDir, - FlagDefault: "./internal/ingest/config", - Required: true, + Name: "captive-core-config-dir", + Usage: "Path to Captive Core's configuration files directory.", + OptType: types.String, + CustomSetValue: utils.SetConfigOptionCaptiveCoreConfigDir, + ConfigKey: &cfg.CaptiveCoreConfigDir, + FlagDefault: "./internal/ingest/config", + Required: true, }, { Name: "ledger-cursor-name", diff --git a/cmd/utils/custom_set_value.go b/cmd/utils/custom_set_value.go index f075a40..37a241f 100644 --- a/cmd/utils/custom_set_value.go +++ b/cmd/utils/custom_set_value.go @@ -1,13 +1,21 @@ package utils import ( + "errors" "fmt" + "os" + "path" "github.com/spf13/viper" "github.com/stellar/go/keypair" "github.com/stellar/go/support/config" + "github.com/stellar/wallet-backend/internal/ingest" ) +func unexpectedTypeError(key any, co *config.ConfigOption) error { + return fmt.Errorf("the expected type for the config key in %s is %T, but a %T was provided instead", co.Name, key, co.ConfigKey) +} + func SetConfigOptionStellarPublicKey(co *config.ConfigOption) error { publicKey := viper.GetString(co.Name) @@ -18,9 +26,61 @@ func SetConfigOptionStellarPublicKey(co *config.ConfigOption) error { key, ok := co.ConfigKey.(*string) if !ok { - return fmt.Errorf("the expected type for the config key in %s is a string, but a %T was provided instead", co.Name, co.ConfigKey) + return unexpectedTypeError(key, co) } *key = kp.Address() return nil } + +func SetConfigOptionCaptiveCoreBinPath(co *config.ConfigOption) error { + binPath := viper.GetString(co.Name) + + fileInfo, err := os.Stat(binPath) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("binary file %s does not exist", binPath) + } + + if fileInfo.IsDir() { + return fmt.Errorf("binary file path %s is a directory, not a file", binPath) + } + + key, ok := co.ConfigKey.(*string) + if !ok { + return unexpectedTypeError(key, co) + } + *key = binPath + + return nil +} + +func SetConfigOptionCaptiveCoreConfigDir(co *config.ConfigOption) error { + dirPath := viper.GetString(co.Name) + + fileInfo, err := os.Stat(dirPath) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("captive core configuration files dir %s does not exist", dirPath) + } + + if !fileInfo.IsDir() { + return fmt.Errorf("captive core configuration files dir %s is not a directory", dirPath) + } + + testnetConfigFile := path.Join(dirPath, ingest.ConfigFileNameTestnet) + if _, err := os.Stat(testnetConfigFile); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("captive core testnet configuration file %s does not exist in dir %s", testnetConfigFile, dirPath) + } + + pubnetConfigFile := path.Join(dirPath, ingest.ConfigFileNamePubnet) + if _, err := os.Stat(pubnetConfigFile); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("captive core pubnet configuration file %s does not exist in dir %s", pubnetConfigFile, dirPath) + } + + key, ok := co.ConfigKey.(*string) + if !ok { + return unexpectedTypeError(key, co) + } + *key = dirPath + + return nil +} diff --git a/cmd/utils/custom_set_value_test.go b/cmd/utils/custom_set_value_test.go index d390d5c..e130238 100644 --- a/cmd/utils/custom_set_value_test.go +++ b/cmd/utils/custom_set_value_test.go @@ -124,3 +124,99 @@ func TestSetConfigOptionStellarPublicKey(t *testing.T) { }) } } + +func TestSetConfigOptionCaptiveCoreBinPath(t *testing.T) { + opts := struct{ binPath string }{} + + co := config.ConfigOption{ + Name: "captive-core-bin-path", + OptType: types.String, + CustomSetValue: SetConfigOptionCaptiveCoreBinPath, + ConfigKey: &opts.binPath, + } + + testCases := []customSetterTestCase[string]{ + { + name: "returns an error if the file path is not set, should be caught by the Require() function", + wantErrContains: "binary file does not exist", + }, + { + name: "returns an error if the path is invalid", + args: []string{"--captive-core-bin-path", "/a/random/path/bin"}, + wantErrContains: "binary file /a/random/path/bin does not exist", + }, + { + name: "returns an error if the path format is invalid", + args: []string{"--captive-core-bin-path", "^7JcrS8J4q@V0$c"}, + wantErrContains: "binary file ^7JcrS8J4q@V0$c does not exist", + }, + { + name: "returns an error if the path is a directory, not a file", + args: []string{"--captive-core-bin-path", "./"}, + wantErrContains: "binary file path ./ is a directory, not a file", + }, + { + name: "sets to ENV var value", + envValue: "./custom_set_value_test.go", + wantResult: "./custom_set_value_test.go", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opts.binPath = "" + customSetterTester(t, tc, co) + }) + } +} + +func TestSetConfigOptionCaptiveCoreConfigDir(t *testing.T) { + opts := struct{ binPath string }{} + + co := config.ConfigOption{ + Name: "captive-core-config-dir", + OptType: types.String, + CustomSetValue: SetConfigOptionCaptiveCoreConfigDir, + ConfigKey: &opts.binPath, + } + + testCases := []customSetterTestCase[string]{ + { + name: "returns an error if the file path is not set, should be caught by the Require() function", + wantErrContains: "captive core configuration files dir does not exist", + }, + { + name: "returns an error if the path is invalid", + envValue: "/a/random/path", + wantErrContains: "captive core configuration files dir /a/random/path does not exist", + }, + { + name: "returns an error if the path format is invalid", + envValue: "^7JcrS8J4q@V0$c", + wantErrContains: "captive core configuration files dir ^7JcrS8J4q@V0$c does not exist", + }, + + { + name: "returns an error if the path is a file, not a directory", + envValue: "./custom_set_value_test.go", + wantErrContains: "captive core configuration files dir ./custom_set_value_test.go is not a directory", + }, + { + name: "returns an error if the directory does not contain the configuration files", + envValue: "./", + wantErrContains: "captive core testnet configuration file stellar-core_testnet.cfg does not exist in dir ./", + }, + { + name: "sets to ENV var value", + envValue: "../../internal/ingest/config", + wantResult: "../../internal/ingest/config", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opts.binPath = "" + customSetterTester(t, tc, co) + }) + } +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index e94a601..8bdd5fe 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -3,6 +3,7 @@ package ingest import ( "context" "fmt" + "path" "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" @@ -12,6 +13,11 @@ import ( "github.com/stellar/wallet-backend/internal/services" ) +const ( + ConfigFileNamePubnet = "stellar-core_pubnet.cfg" + ConfigFileNameTestnet = "stellar-core_testnet.cfg" +) + type Configs struct { DatabaseURL string NetworkPassphrase string @@ -73,10 +79,10 @@ func getCaptiveCoreConfig(cfg Configs) (ledgerbackend.CaptiveCoreConfig, error) switch cfg.NetworkPassphrase { case network.TestNetworkPassphrase: networkArchivesURLs = network.TestNetworkhistoryArchiveURLs - configFilePath = cfg.CaptiveCoreConfigDir + "/stellar-core_testnet.cfg" + configFilePath = path.Join(cfg.CaptiveCoreConfigDir, ConfigFileNameTestnet) case network.PublicNetworkPassphrase: networkArchivesURLs = network.PublicNetworkhistoryArchiveURLs - configFilePath = cfg.CaptiveCoreConfigDir + "/stellar-core_pubnet.cfg" + configFilePath = path.Join(cfg.CaptiveCoreConfigDir, ConfigFileNamePubnet) default: return ledgerbackend.CaptiveCoreConfig{}, fmt.Errorf("unknown network: %s", cfg.NetworkPassphrase) } From 72a0dbbeeffb8a4c2d127edc74476bd9c69fdf16 Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Mon, 27 May 2024 11:09:05 -0300 Subject: [PATCH 06/14] rework: ledger-cursor-name made more explicit --- cmd/ingest.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cmd/ingest.go b/cmd/ingest.go index 73bf81f..6dc95b4 100644 --- a/cmd/ingest.go +++ b/cmd/ingest.go @@ -52,12 +52,11 @@ func (c *ingestCmd) Command() *cobra.Command { Required: true, }, { - Name: "ledger-cursor-name", - Usage: "Name of last synced ledger cursor. Attention: there should never be more than one container running with a same cursor name.", - OptType: types.String, - ConfigKey: &cfg.LedgerCursorName, - FlagDefault: "last_synced_ledger", - Required: false, + Name: "ledger-cursor-name", + Usage: "Name of last synced ledger cursor, used to keep track of the last ledger ingested by the service. When starting up, ingestion will resume from the ledger number stored in this record. It should be an unique name per container as different containers would overwrite the cursor value of its peers when using the name cursor name.", + OptType: types.String, + ConfigKey: &cfg.LedgerCursorName, + Required: true, }, { Name: "start", From 30ec11a209ecd493b76376de1f91ed3e86d30eb8 Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Mon, 27 May 2024 14:43:45 -0300 Subject: [PATCH 07/14] rework: log level moved to custom set value serve --- cmd/ingest.go | 9 ++++++ cmd/root.go | 2 -- cmd/serve.go | 9 ++++++ cmd/utils/custom_set_value.go | 26 +++++++++++++++ cmd/utils/custom_set_value_test.go | 52 ++++++++++++++++++++++++++++++ internal/ingest/ingest.go | 2 ++ internal/serve/serve.go | 2 ++ 7 files changed, 100 insertions(+), 2 deletions(-) diff --git a/cmd/ingest.go b/cmd/ingest.go index 6dc95b4..7d53fc8 100644 --- a/cmd/ingest.go +++ b/cmd/ingest.go @@ -33,6 +33,15 @@ func (c *ingestCmd) Command() *cobra.Command { FlagDefault: network.TestNetworkPassphrase, Required: true, }, + { + Name: "log-level", + Usage: `The log level used in this project. Options: "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL", or "PANIC".`, + OptType: types.String, + FlagDefault: "TRACE", + ConfigKey: &cfg.LogLevel, + CustomSetValue: utils.SetConfigOptionLogLevel, + Required: false, + }, { Name: "captive-core-bin-path", Usage: "Path to Captive Core's binary file.", diff --git a/cmd/root.go b/cmd/root.go index 3fc1ade..1e3967d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,7 +1,6 @@ package cmd import ( - "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/stellar/go/support/log" ) @@ -29,7 +28,6 @@ func Execute() { func init() { log.DefaultLogger = log.New() - log.DefaultLogger.SetLevel(logrus.TraceLevel) rootCmd.AddCommand((&serveCmd{}).Command()) rootCmd.AddCommand((&ingestCmd{}).Command()) diff --git a/cmd/serve.go b/cmd/serve.go index a8422cd..c6a8bd2 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -40,6 +40,15 @@ func (c *serveCmd) Command() *cobra.Command { FlagDefault: "http://localhost:8000", Required: true, }, + { + Name: "log-level", + Usage: `The log level used in this project. Options: "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL", or "PANIC".`, + OptType: types.String, + FlagDefault: "TRACE", + ConfigKey: &cfg.LogLevel, + CustomSetValue: utils.SetConfigOptionLogLevel, + Required: false, + }, { Name: "wallet-signing-key", Usage: "The public key of the Stellar account that signs the payloads when making HTTP Request to this server.", diff --git a/cmd/utils/custom_set_value.go b/cmd/utils/custom_set_value.go index 37a241f..c702dff 100644 --- a/cmd/utils/custom_set_value.go +++ b/cmd/utils/custom_set_value.go @@ -6,9 +6,11 @@ import ( "os" "path" + "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/stellar/go/keypair" "github.com/stellar/go/support/config" + "github.com/stellar/go/support/log" "github.com/stellar/wallet-backend/internal/ingest" ) @@ -16,6 +18,30 @@ func unexpectedTypeError(key any, co *config.ConfigOption) error { return fmt.Errorf("the expected type for the config key in %s is %T, but a %T was provided instead", co.Name, key, co.ConfigKey) } +func SetConfigOptionLogLevel(co *config.ConfigOption) error { + logLevelStr := viper.GetString(co.Name) + logLevel, err := logrus.ParseLevel(logLevelStr) + if err != nil { + return fmt.Errorf("couldn't parse log level in %s: %w", co.Name, err) + } + + key, ok := co.ConfigKey.(*logrus.Level) + if !ok { + return fmt.Errorf("%s configKey has an invalid type %T", co.Name, co.ConfigKey) + } + *key = logLevel + + // Log for debugging + if config.IsExplicitlySet(co) { + log.Debugf("Setting log level to: %s", logLevel) + log.DefaultLogger.SetLevel(*key) + } else { + log.Debugf("Using default log level: %s", logLevel) + } + + return nil +} + func SetConfigOptionStellarPublicKey(co *config.ConfigOption) error { publicKey := viper.GetString(co.Name) diff --git a/cmd/utils/custom_set_value_test.go b/cmd/utils/custom_set_value_test.go index e130238..0d15ecc 100644 --- a/cmd/utils/custom_set_value_test.go +++ b/cmd/utils/custom_set_value_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/stellar/go/support/config" "github.com/stellar/wallet-backend/internal/utils" @@ -125,6 +126,57 @@ func TestSetConfigOptionStellarPublicKey(t *testing.T) { } } +func Test_SetConfigOptionLogLevel(t *testing.T) { + opts := struct{ logrusLevel logrus.Level }{} + + co := config.ConfigOption{ + Name: "log-level", + OptType: types.String, + CustomSetValue: SetConfigOptionLogLevel, + ConfigKey: &opts.logrusLevel, + } + + testCases := []customSetterTestCase[logrus.Level]{ + { + name: "returns an error if the log level is empty", + args: []string{}, + wantErrContains: `couldn't parse log level in log-level: not a valid logrus Level: ""`, + }, + { + name: "returns an error if the log level is invalid", + args: []string{"--log-level", "test"}, + wantErrContains: `couldn't parse log level in log-level: not a valid logrus Level: "test"`, + }, + { + name: "handles messenger type TRACE (through CLI args)", + args: []string{"--log-level", "TRACE"}, + wantResult: logrus.TraceLevel, + }, + { + name: "handles messenger type TRACE (through ENV vars)", + envValue: "TRACE", + wantResult: logrus.TraceLevel, + }, + { + name: "handles messenger type INFO (through CLI args)", + args: []string{"--log-level", "iNfO"}, + wantResult: logrus.InfoLevel, + }, + { + name: "handles messenger type INFO (through ENV vars)", + envValue: "INFO", + wantResult: logrus.InfoLevel, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opts.logrusLevel = 0 + customSetterTester[logrus.Level](t, tc, co) + }) + } +} + func TestSetConfigOptionCaptiveCoreBinPath(t *testing.T) { opts := struct{ binPath string }{} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 8bdd5fe..43910b2 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -5,6 +5,7 @@ import ( "fmt" "path" + "github.com/sirupsen/logrus" "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" "github.com/stellar/go/support/log" @@ -26,6 +27,7 @@ type Configs struct { LedgerCursorName string StartLedger int EndLedger int + LogLevel logrus.Level } func Ingest(cfg Configs) error { diff --git a/internal/serve/serve.go b/internal/serve/serve.go index 1ade460..a2d2326 100644 --- a/internal/serve/serve.go +++ b/internal/serve/serve.go @@ -5,6 +5,7 @@ import ( "net/http" "github.com/go-chi/chi" + "github.com/sirupsen/logrus" supporthttp "github.com/stellar/go/support/http" "github.com/stellar/go/support/log" "github.com/stellar/go/support/render/health" @@ -21,6 +22,7 @@ type Configs struct { DatabaseURL string ServerBaseURL string WalletSigningKey string + LogLevel logrus.Level } type handlerDeps struct { From 573569389d9bccf3c6b45767ea1611288857f555 Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Mon, 27 May 2024 18:28:52 -0300 Subject: [PATCH 08/14] rework: implement runintransaction wrapper --- internal/data/models.go | 2 +- internal/data/payments.go | 23 ++----- internal/data/payments_test.go | 16 ++--- internal/db/db.go | 112 +++++++++++++++++++++++++++++---- internal/db/db_test.go | 4 +- internal/services/ingest.go | 107 ++++++++++++++----------------- 6 files changed, 163 insertions(+), 101 deletions(-) diff --git a/internal/data/models.go b/internal/data/models.go index 2da7db6..fbd9c5d 100644 --- a/internal/data/models.go +++ b/internal/data/models.go @@ -16,6 +16,6 @@ func NewModels(db db.ConnectionPool) (*Models, error) { } return &Models{ - Payments: &PaymentModel{db: db}, + Payments: &PaymentModel{DB: db}, }, nil } diff --git a/internal/data/payments.go b/internal/data/payments.go index d8a4395..7ef2bb9 100644 --- a/internal/data/payments.go +++ b/internal/data/payments.go @@ -7,12 +7,10 @@ import ( "time" "github.com/stellar/wallet-backend/internal/db" - - "github.com/jmoiron/sqlx" ) type PaymentModel struct { - db db.ConnectionPool + DB db.ConnectionPool } type Payment struct { @@ -34,7 +32,7 @@ type Payment struct { func (m *PaymentModel) GetLatestLedgerSynced(ctx context.Context, cursorName string) (uint32, error) { var lastSyncedLedger uint32 - err := m.db.QueryRowxContext(ctx, `SELECT value FROM ingest_store WHERE key = $1`, cursorName).Scan(&lastSyncedLedger) + err := m.DB.QueryRowxContext(ctx, `SELECT value FROM ingest_store WHERE key = $1`, cursorName).Scan(&lastSyncedLedger) // First run, key does not exist yet if err == sql.ErrNoRows { return 0, nil @@ -51,7 +49,7 @@ func (m *PaymentModel) UpdateLatestLedgerSynced(ctx context.Context, cursorName INSERT INTO ingest_store (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = excluded.value ` - _, err := m.db.ExecContext(ctx, query, cursorName, ledger) + _, err := m.DB.ExecContext(ctx, query, cursorName, ledger) if err != nil { return fmt.Errorf("updating last synced ledger to %d: %w", ledger, err) } @@ -59,16 +57,7 @@ func (m *PaymentModel) UpdateLatestLedgerSynced(ctx context.Context, cursorName return nil } -func (m *PaymentModel) BeginTx(ctx context.Context) (*sqlx.Tx, error) { - tx, err := m.db.BeginTxx(ctx, nil) - if err != nil { - return nil, fmt.Errorf("beginning transaction: %w", err) - } - - return tx, nil -} - -func (m *PaymentModel) AddPayment(ctx context.Context, tx *sqlx.Tx, payment Payment) error { +func (m *PaymentModel) AddPayment(ctx context.Context, tx db.Transaction, payment Payment) error { const query = ` INSERT INTO ingest_payments ( operation_id, operation_type, transaction_id, transaction_hash, from_address, to_address, src_asset_code, src_asset_issuer, src_amount, @@ -105,7 +94,7 @@ func (m *PaymentModel) AddPayment(ctx context.Context, tx *sqlx.Tx, payment Paym func (m *PaymentModel) SubscribeAddress(ctx context.Context, address string) error { const query = `INSERT INTO accounts (stellar_address) VALUES ($1) ON CONFLICT DO NOTHING` - _, err := m.db.ExecContext(ctx, query, address) + _, err := m.DB.ExecContext(ctx, query, address) if err != nil { return fmt.Errorf("subscribing address %s to payments tracking: %w", address, err) } @@ -115,7 +104,7 @@ func (m *PaymentModel) SubscribeAddress(ctx context.Context, address string) err func (m *PaymentModel) UnsubscribeAddress(ctx context.Context, address string) error { const query = `DELETE FROM accounts WHERE stellar_address = $1` - _, err := m.db.ExecContext(ctx, query, address) + _, err := m.DB.ExecContext(ctx, query, address) if err != nil { return fmt.Errorf("unsubscribing address %s to payments tracking: %w", address, err) } diff --git a/internal/data/payments_test.go b/internal/data/payments_test.go index 03d78c3..63e77f5 100644 --- a/internal/data/payments_test.go +++ b/internal/data/payments_test.go @@ -22,7 +22,7 @@ func TestAddPayment(t *testing.T) { defer dbConnectionPool.Close() m := &PaymentModel{ - db: dbConnectionPool, + DB: dbConnectionPool, } ctx := context.Background() @@ -48,13 +48,9 @@ func TestAddPayment(t *testing.T) { } addPayment := func() { - tx, err := m.BeginTx(ctx) - require.NoError(t, err) - - err = m.AddPayment(ctx, tx, payment) - require.NoError(t, err) - - err = tx.Commit() + err := db.RunInTransaction(ctx, m.DB, nil, func(dbTx db.Transaction) error { + return m.AddPayment(ctx, dbTx, payment) + }) require.NoError(t, err) } @@ -114,7 +110,7 @@ func TestSubscribeAddress(t *testing.T) { defer dbConnectionPool.Close() m := &PaymentModel{ - db: dbConnectionPool, + DB: dbConnectionPool, } ctx := context.Background() @@ -123,7 +119,7 @@ func TestSubscribeAddress(t *testing.T) { require.NoError(t, err) var dbAddress sql.NullString - err = m.db.GetContext(ctx, &dbAddress, "SELECT stellar_address FROM accounts LIMIT 1") + err = m.DB.GetContext(ctx, &dbAddress, "SELECT stellar_address FROM accounts LIMIT 1") require.NoError(t, err) assert.True(t, dbAddress.Valid) diff --git a/internal/db/db.go b/internal/db/db.go index ca7773b..264aaa8 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -7,24 +7,22 @@ import ( "time" "github.com/jmoiron/sqlx" + "github.com/stellar/go/support/log" ) type ConnectionPool interface { + SQLExecuter + BeginTxx(ctx context.Context, opts *sql.TxOptions) (Transaction, error) Close() error - Ping() error - DriverName() string - sqlx.ExecerContext - sqlx.QueryerContext - sqlx.PreparerContext - GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error - SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error - BeginTxx(ctx context.Context, opts *sql.TxOptions) (*sqlx.Tx, error) + Ping(ctx context.Context) error + SqlDB(ctx context.Context) (*sql.DB, error) + SqlxDB(ctx context.Context) (*sqlx.DB, error) } // Make sure *DBConnectionPoolImplementation implements DBConnectionPool: -var _ ConnectionPool = (*DBConnectionPoolImplementation)(nil) +var _ ConnectionPool = (*ConnectionPoolImplementation)(nil) -type DBConnectionPoolImplementation struct { +type ConnectionPoolImplementation struct { *sqlx.DB } @@ -46,5 +44,97 @@ func OpenDBConnectionPool(dataSourceName string) (ConnectionPool, error) { return nil, fmt.Errorf("error pinging app DB connection pool: %w", err) } - return &DBConnectionPoolImplementation{DB: sqlxDB}, nil + return &ConnectionPoolImplementation{DB: sqlxDB}, nil +} + +func (db *ConnectionPoolImplementation) BeginTxx(ctx context.Context, opts *sql.TxOptions) (Transaction, error) { + return db.DB.BeginTxx(ctx, opts) +} + +func (db *ConnectionPoolImplementation) Ping(ctx context.Context) error { + return db.DB.PingContext(ctx) +} + +func (db *ConnectionPoolImplementation) SqlDB(ctx context.Context) (*sql.DB, error) { + return db.DB.DB, nil +} + +func (db *ConnectionPoolImplementation) SqlxDB(ctx context.Context) (*sqlx.DB, error) { + return db.DB, nil +} + +// Transaction is an interface that wraps the sqlx.Tx structs methods. +type Transaction interface { + SQLExecuter + Rollback() error + Commit() error +} + +// Make sure *sqlx.Tx implements DBTransaction: +var _ Transaction = (*sqlx.Tx)(nil) + +// SQLExecuter is an interface that wraps the *sqlx.DB and *sqlx.Tx structs methods. +type SQLExecuter interface { + DriverName() string + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) + GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error + sqlx.PreparerContext + sqlx.QueryerContext + Rebind(query string) string + SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error +} + +// Make sure *sqlx.DB implements SQLExecuter: +var _ SQLExecuter = (*sqlx.DB)(nil) + +// Make sure DBConnectionPool implements SQLExecuter: +var _ SQLExecuter = (ConnectionPool)(nil) + +// Make sure *sqlx.Tx implements SQLExecuter: +var _ SQLExecuter = (*sqlx.Tx)(nil) + +// Make sure DBTransaction implements SQLExecuter: +var _ SQLExecuter = (Transaction)(nil) + +// RunInTransaction runs the given atomic function in an atomic database transaction and returns an error. Boilerplate +// code for database transactions. +func RunInTransaction(ctx context.Context, dbConnectionPool ConnectionPool, opts *sql.TxOptions, atomicFunction func(dbTx Transaction) error) error { + // wrap the atomic function with a function that returns nil and an error so we can call RunInTransactionWithResult + wrappedFunction := func(dbTx Transaction) (interface{}, error) { + return nil, atomicFunction(dbTx) + } + + _, err := RunInTransactionWithResult(ctx, dbConnectionPool, opts, wrappedFunction) + return err +} + +// RunInTransactionWithResult runs the given atomic function in an atomic database transaction and returns a result and +// an error. Boilerplate code for database transactions. +func RunInTransactionWithResult[T any](ctx context.Context, dbConnectionPool ConnectionPool, opts *sql.TxOptions, atomicFunction func(dbTx Transaction) (T, error)) (result T, err error) { + dbTx, err := dbConnectionPool.BeginTxx(ctx, opts) + if err != nil { + return *new(T), fmt.Errorf("creating db transaction for RunInTransactionWithResult: %w", err) + } + + defer func() { + if err != nil { + log.Ctx(ctx).Errorf("Rolling back transaction due to error: %v", err) + errRollBack := dbTx.Rollback() + if errRollBack != nil { + log.Ctx(ctx).Errorf("Error in database transaction rollback: %v", errRollBack) + } + } + }() + + result, err = atomicFunction(dbTx) + if err != nil { + return *new(T), fmt.Errorf("running atomic function in RunInTransactionWithResult: %w", err) + } + + err = dbTx.Commit() + if err != nil { + return *new(T), fmt.Errorf("committing transaction in RunInTransactionWithResult: %w", err) + } + + return result, nil } diff --git a/internal/db/db_test.go b/internal/db/db_test.go index ee64545..d2cbb2f 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -1,6 +1,7 @@ package db import ( + "context" "testing" "github.com/stellar/go/support/db/dbtest" @@ -18,6 +19,7 @@ func TestOpenDBConnectionPool(t *testing.T) { assert.Equal(t, "postgres", dbConnectionPool.DriverName()) - err = dbConnectionPool.Ping() + ctx := context.Background() + err = dbConnectionPool.Ping(ctx) require.NoError(t, err) } diff --git a/internal/services/ingest.go b/internal/services/ingest.go index 50c9312..7c23746 100644 --- a/internal/services/ingest.go +++ b/internal/services/ingest.go @@ -12,6 +12,7 @@ import ( "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" "github.com/stellar/wallet-backend/internal/data" + "github.com/stellar/wallet-backend/internal/db" "github.com/stellar/wallet-backend/internal/utils" ) @@ -122,77 +123,61 @@ func (m *IngestManager) processLedger(ctx context.Context, ledger uint32, ledger ledgerCloseTime := time.Unix(int64(ledgerMeta.LedgerHeaderHistoryEntry().Header.ScpValue.CloseTime), 0).UTC() ledgerSequence := ledgerMeta.LedgerSequence() - dbTx, err := m.PaymentModel.BeginTx(ctx) - if err != nil { - return err - } - defer func() { - if err != nil { - rollbackErr := dbTx.Rollback() - if rollbackErr != nil { - log.Ctx(ctx).Error(rollbackErr) + return db.RunInTransaction(ctx, m.PaymentModel.DB, nil, func(dbTx db.Transaction) error { + for { + tx, err := reader.Read() + if err == io.EOF { + break } - } - }() - - for { - tx, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - return fmt.Errorf("reading transaction: %w", err) - } - - if !tx.Result.Successful() { - continue - } - - txHash := utils.TransactionHash(ledgerMeta, int(tx.Index)) - txMemo := utils.Memo(tx.Envelope.Memo(), txHash) - - for idx, op := range tx.Envelope.Operations() { - opIdx := idx + 1 - - payment := data.Payment{ - OperationID: utils.OperationID(int32(ledgerSequence), int32(tx.Index), int32(opIdx)), - OperationType: op.Body.Type.String(), - TransactionID: utils.TransactionID(int32(ledgerSequence), int32(tx.Index)), - TransactionHash: txHash, - From: utils.SourceAccount(op, tx), - CreatedAt: ledgerCloseTime, - Memo: utils.SanitizeUTF8(txMemo), // Field is subject to user input + if err != nil { + return fmt.Errorf("reading transaction: %w", err) } - switch op.Body.Type { - case xdr.OperationTypePayment: - fillPayment(&payment, op.Body) - case xdr.OperationTypePathPaymentStrictSend: - fillPathSend(&payment, op.Body, tx, opIdx) - case xdr.OperationTypePathPaymentStrictReceive: - fillPathReceive(&payment, op.Body, tx, opIdx) - default: + if !tx.Result.Successful() { continue } - err = m.PaymentModel.AddPayment(ctx, dbTx, payment) - if err != nil { - return fmt.Errorf("adding payment for ledger %d, tx %q (%d), operation %d (%d): %w", ledgerSequence, txHash, tx.Index, payment.OperationID, opIdx, err) + txHash := utils.TransactionHash(ledgerMeta, int(tx.Index)) + txMemo := utils.Memo(tx.Envelope.Memo(), txHash) + + for idx, op := range tx.Envelope.Operations() { + opIdx := idx + 1 + + payment := data.Payment{ + OperationID: utils.OperationID(int32(ledgerSequence), int32(tx.Index), int32(opIdx)), + OperationType: op.Body.Type.String(), + TransactionID: utils.TransactionID(int32(ledgerSequence), int32(tx.Index)), + TransactionHash: txHash, + From: utils.SourceAccount(op, tx), + CreatedAt: ledgerCloseTime, + Memo: utils.SanitizeUTF8(txMemo), // Field is subject to user input + } + + switch op.Body.Type { + case xdr.OperationTypePayment: + fillPayment(&payment, op.Body) + case xdr.OperationTypePathPaymentStrictSend: + fillPathSend(&payment, op.Body, tx, opIdx) + case xdr.OperationTypePathPaymentStrictReceive: + fillPathReceive(&payment, op.Body, tx, opIdx) + default: + continue + } + + err = m.PaymentModel.AddPayment(ctx, dbTx, payment) + if err != nil { + return fmt.Errorf("adding payment for ledger %d, tx %q (%d), operation %d (%d): %w", ledgerSequence, txHash, tx.Index, payment.OperationID, opIdx, err) + } } } - } - - err = m.PaymentModel.UpdateLatestLedgerSynced(ctx, m.LedgerCursorName, ledger) - if err != nil { - return err - } - err = dbTx.Commit() - if err != nil { - return err - } + err = m.PaymentModel.UpdateLatestLedgerSynced(ctx, m.LedgerCursorName, ledger) + if err != nil { + return err + } - return nil + return nil + }) } func fillPayment(payment *data.Payment, operation xdr.OperationBody) { From f951c14d771e7e4c362287a19975e93707169865 Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Mon, 27 May 2024 18:29:07 -0300 Subject: [PATCH 09/14] rework: payments model --- internal/data/payments.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/internal/data/payments.go b/internal/data/payments.go index 7ef2bb9..5d534f2 100644 --- a/internal/data/payments.go +++ b/internal/data/payments.go @@ -14,20 +14,20 @@ type PaymentModel struct { } type Payment struct { - OperationID int64 - OperationType string - TransactionID int64 - TransactionHash string - From string - To string - SrcAssetCode string - SrcAssetIssuer string - SrcAmount int64 - DestAssetCode string - DestAssetIssuer string - DestAmount int64 - CreatedAt time.Time - Memo string + OperationID int64 `db:"operation_id"` + OperationType string `db:"operation_type"` + TransactionID int64 `db:"transaction_id"` + TransactionHash string `db:"transaction_hash"` + From string `db:"from"` + To string `db:"to"` + SrcAssetCode string `db:"src_asset_code"` + SrcAssetIssuer string `db:"src_asset_issuer"` + SrcAmount int64 `db:"src_amount"` + DestAssetCode string `db:"dest_asset_code"` + DestAssetIssuer string `db:"dest_asset_issuer"` + DestAmount int64 `db:"dest_amount"` + CreatedAt time.Time `db:"created_at"` + Memo string `db:"memo"` } func (m *PaymentModel) GetLatestLedgerSynced(ctx context.Context, cursorName string) (uint32, error) { @@ -38,7 +38,7 @@ func (m *PaymentModel) GetLatestLedgerSynced(ctx context.Context, cursorName str return 0, nil } if err != nil { - return 0, err + return 0, fmt.Errorf("getting latest ledger synced for cursor %s: %w", cursorName, err) } return lastSyncedLedger, nil From 65edfa1990c3966e6271b831f3738c4a2445dadd Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Tue, 28 May 2024 09:32:54 -0300 Subject: [PATCH 10/14] rework: getCaptiveCoreConfig tests --- cmd/utils/custom_set_value.go | 12 ------ cmd/utils/custom_set_value_test.go | 5 --- internal/ingest/ingest.go | 20 ++++++---- internal/ingest/ingest_test.go | 61 ++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 24 deletions(-) create mode 100644 internal/ingest/ingest_test.go diff --git a/cmd/utils/custom_set_value.go b/cmd/utils/custom_set_value.go index c702dff..97ef80b 100644 --- a/cmd/utils/custom_set_value.go +++ b/cmd/utils/custom_set_value.go @@ -4,14 +4,12 @@ import ( "errors" "fmt" "os" - "path" "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/stellar/go/keypair" "github.com/stellar/go/support/config" "github.com/stellar/go/support/log" - "github.com/stellar/wallet-backend/internal/ingest" ) func unexpectedTypeError(key any, co *config.ConfigOption) error { @@ -92,16 +90,6 @@ func SetConfigOptionCaptiveCoreConfigDir(co *config.ConfigOption) error { return fmt.Errorf("captive core configuration files dir %s is not a directory", dirPath) } - testnetConfigFile := path.Join(dirPath, ingest.ConfigFileNameTestnet) - if _, err := os.Stat(testnetConfigFile); errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("captive core testnet configuration file %s does not exist in dir %s", testnetConfigFile, dirPath) - } - - pubnetConfigFile := path.Join(dirPath, ingest.ConfigFileNamePubnet) - if _, err := os.Stat(pubnetConfigFile); errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("captive core pubnet configuration file %s does not exist in dir %s", pubnetConfigFile, dirPath) - } - key, ok := co.ConfigKey.(*string) if !ok { return unexpectedTypeError(key, co) diff --git a/cmd/utils/custom_set_value_test.go b/cmd/utils/custom_set_value_test.go index 0d15ecc..286d720 100644 --- a/cmd/utils/custom_set_value_test.go +++ b/cmd/utils/custom_set_value_test.go @@ -253,11 +253,6 @@ func TestSetConfigOptionCaptiveCoreConfigDir(t *testing.T) { envValue: "./custom_set_value_test.go", wantErrContains: "captive core configuration files dir ./custom_set_value_test.go is not a directory", }, - { - name: "returns an error if the directory does not contain the configuration files", - envValue: "./", - wantErrContains: "captive core testnet configuration file stellar-core_testnet.cfg does not exist in dir ./", - }, { name: "sets to ENV var value", envValue: "../../internal/ingest/config", diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 43910b2..33a6b88 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -2,7 +2,9 @@ package ingest import ( "context" + "errors" "fmt" + "os" "path" "github.com/sirupsen/logrus" @@ -14,11 +16,6 @@ import ( "github.com/stellar/wallet-backend/internal/services" ) -const ( - ConfigFileNamePubnet = "stellar-core_pubnet.cfg" - ConfigFileNameTestnet = "stellar-core_testnet.cfg" -) - type Configs struct { DatabaseURL string NetworkPassphrase string @@ -74,6 +71,11 @@ func setupDeps(cfg Configs) (*services.IngestManager, error) { }, nil } +const ( + configFileNamePubnet = "stellar-core_pubnet.cfg" + configFileNameTestnet = "stellar-core_testnet.cfg" +) + func getCaptiveCoreConfig(cfg Configs) (ledgerbackend.CaptiveCoreConfig, error) { var networkArchivesURLs []string var configFilePath string @@ -81,14 +83,18 @@ func getCaptiveCoreConfig(cfg Configs) (ledgerbackend.CaptiveCoreConfig, error) switch cfg.NetworkPassphrase { case network.TestNetworkPassphrase: networkArchivesURLs = network.TestNetworkhistoryArchiveURLs - configFilePath = path.Join(cfg.CaptiveCoreConfigDir, ConfigFileNameTestnet) + configFilePath = path.Join(cfg.CaptiveCoreConfigDir, configFileNameTestnet) case network.PublicNetworkPassphrase: networkArchivesURLs = network.PublicNetworkhistoryArchiveURLs - configFilePath = path.Join(cfg.CaptiveCoreConfigDir, ConfigFileNamePubnet) + configFilePath = path.Join(cfg.CaptiveCoreConfigDir, configFileNamePubnet) default: return ledgerbackend.CaptiveCoreConfig{}, fmt.Errorf("unknown network: %s", cfg.NetworkPassphrase) } + if _, err := os.Stat(configFilePath); errors.Is(err, os.ErrNotExist) { + return ledgerbackend.CaptiveCoreConfig{}, fmt.Errorf("captive core configuration file not found in %s", configFilePath) + } + // Read configuration TOML captiveCoreToml, err := ledgerbackend.NewCaptiveCoreTomlFromFile(configFilePath, ledgerbackend.CaptiveCoreTomlParams{ CoreBinaryPath: cfg.CaptiveCoreBinPath, diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go new file mode 100644 index 0000000..5bf4c59 --- /dev/null +++ b/internal/ingest/ingest_test.go @@ -0,0 +1,61 @@ +package ingest + +import ( + "testing" + + "github.com/stellar/go/network" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetCaptiveCoreConfig(t *testing.T) { + t.Run("testnet_success", func(t *testing.T) { + config, err := getCaptiveCoreConfig(Configs{ + NetworkPassphrase: network.TestNetworkPassphrase, + CaptiveCoreBinPath: "/bin/path", + CaptiveCoreConfigDir: "./config", + }) + + require.NoError(t, err) + assert.Equal(t, "/bin/path", config.BinaryPath) + assert.Equal(t, network.TestNetworkPassphrase, config.NetworkPassphrase) + assert.Equal(t, network.TestNetworkhistoryArchiveURLs, config.HistoryArchiveURLs) + assert.Equal(t, true, config.UseDB) + assert.NotNil(t, config.Toml) + }) + + t.Run("pubnet_success", func(t *testing.T) { + config, err := getCaptiveCoreConfig(Configs{ + NetworkPassphrase: network.PublicNetworkPassphrase, + CaptiveCoreBinPath: "/bin/path", + CaptiveCoreConfigDir: "./config", + }) + + require.NoError(t, err) + assert.Equal(t, "/bin/path", config.BinaryPath) + assert.Equal(t, network.PublicNetworkPassphrase, config.NetworkPassphrase) + assert.Equal(t, network.PublicNetworkhistoryArchiveURLs, config.HistoryArchiveURLs) + assert.Equal(t, true, config.UseDB) + assert.NotNil(t, config.Toml) + }) + + t.Run("unknown_network", func(t *testing.T) { + _, err := getCaptiveCoreConfig(Configs{ + NetworkPassphrase: "Invalid SDF Network ; May 2024", + CaptiveCoreBinPath: "/bin/path", + CaptiveCoreConfigDir: "./config", + }) + + assert.ErrorContains(t, err, "unknown network: Invalid SDF Network ; May 2024") + }) + + t.Run("invalid_config_file", func(t *testing.T) { + _, err := getCaptiveCoreConfig(Configs{ + NetworkPassphrase: network.TestNetworkPassphrase, + CaptiveCoreBinPath: "/bin/path", + CaptiveCoreConfigDir: "./invalid/path", + }) + + assert.ErrorContains(t, err, "captive core configuration file not found in invalid/path/stellar-core_testnet.cfg") + }) +} From eb02874099e26dcba3e15945cd6f18049c3d80a4 Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Tue, 28 May 2024 11:02:49 -0300 Subject: [PATCH 11/14] tests added --- internal/utils/ingestion_utils.go | 4 +- internal/utils/ingestion_utils_test.go | 68 ++++++++++++++++++++++++++ internal/utils/utils_test.go | 12 +++++ 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 internal/utils/ingestion_utils_test.go diff --git a/internal/utils/ingestion_utils.go b/internal/utils/ingestion_utils.go index efebad3..68cf2b6 100644 --- a/internal/utils/ingestion_utils.go +++ b/internal/utils/ingestion_utils.go @@ -12,8 +12,8 @@ func OperationID(ledgerNumber, txNumber, opNumber int32) int64 { return toid.New(ledgerNumber, txNumber, opNumber).ToInt64() } -func OperationResult(transaction ingest.LedgerTransaction, opNumber int) *xdr.OperationResultTr { - results, _ := transaction.Result.OperationResults() +func OperationResult(tx ingest.LedgerTransaction, opNumber int) *xdr.OperationResultTr { + results, _ := tx.Result.OperationResults() tr := results[opNumber-1].MustTr() return &tr } diff --git a/internal/utils/ingestion_utils_test.go b/internal/utils/ingestion_utils_test.go new file mode 100644 index 0000000..d689d98 --- /dev/null +++ b/internal/utils/ingestion_utils_test.go @@ -0,0 +1,68 @@ +package utils + +import ( + "crypto/sha256" + "testing" + + "github.com/stellar/go/xdr" + "github.com/stretchr/testify/assert" +) + +func TestMemo(t *testing.T) { + t.Run("type_none", func(t *testing.T) { + memo := xdr.Memo{ + Type: xdr.MemoTypeMemoNone, + } + + result := Memo(memo, "") + assert.Equal(t, "", result) + }) + + t.Run("type_text", func(t *testing.T) { + value := "test" + memo := xdr.Memo{ + Type: xdr.MemoTypeMemoText, + Text: &value, + } + + result := Memo(memo, "") + assert.Equal(t, "test", result) + }) + + t.Run("type_id", func(t *testing.T) { + value := xdr.Uint64(12345) + memo := xdr.Memo{ + Type: xdr.MemoTypeMemoId, + Id: &value, + } + + result := Memo(memo, "") + assert.Equal(t, "12345", result) + }) + + t.Run("type_hash", func(t *testing.T) { + hash := sha256.New() + hash.Write([]byte("test")) + value := xdr.Hash(hash.Sum(nil)) + memo := xdr.Memo{ + Type: xdr.MemoTypeMemoHash, + Hash: &value, + } + + result := Memo(memo, "") + assert.Equal(t, value.HexString(), result) + }) + + t.Run("type_return", func(t *testing.T) { + hash := sha256.New() + hash.Write([]byte("test")) + value := xdr.Hash(hash.Sum(nil)) + memo := xdr.Memo{ + Type: xdr.MemoTypeMemoReturn, + RetHash: &value, + } + + result := Memo(memo, "") + assert.Equal(t, value.HexString(), result) + }) +} diff --git a/internal/utils/utils_test.go b/internal/utils/utils_test.go index 557d025..8c75b79 100644 --- a/internal/utils/utils_test.go +++ b/internal/utils/utils_test.go @@ -6,6 +6,18 @@ import ( "github.com/stretchr/testify/assert" ) +func TestSanitizeUTF8(t *testing.T) { + t.Run("non_utf8", func(t *testing.T) { + result := SanitizeUTF8("test\xF5") + assert.Equal(t, "test?", result) + }) + + t.Run("zero_byte", func(t *testing.T) { + result := SanitizeUTF8("test\x00\x00") + assert.Equal(t, "test", result) + }) +} + func TestUnwrapInterfaceToPointer(t *testing.T) { // Test with a string strValue := "test" From ac1f46ea963cc6657b6329ff0fbc4f4d32ee8d26 Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Tue, 28 May 2024 19:12:45 -0300 Subject: [PATCH 12/14] test added: processLedger --- internal/data/payments.go | 4 +- internal/services/ingest_test.go | 142 +++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 internal/services/ingest_test.go diff --git a/internal/data/payments.go b/internal/data/payments.go index 5d534f2..603d009 100644 --- a/internal/data/payments.go +++ b/internal/data/payments.go @@ -18,8 +18,8 @@ type Payment struct { OperationType string `db:"operation_type"` TransactionID int64 `db:"transaction_id"` TransactionHash string `db:"transaction_hash"` - From string `db:"from"` - To string `db:"to"` + From string `db:"from_address"` + To string `db:"to_address"` SrcAssetCode string `db:"src_asset_code"` SrcAssetIssuer string `db:"src_asset_issuer"` SrcAmount int64 `db:"src_amount"` diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go new file mode 100644 index 0000000..327552d --- /dev/null +++ b/internal/services/ingest_test.go @@ -0,0 +1,142 @@ +package services + +import ( + "context" + "testing" + "time" + + "github.com/stellar/go/network" + "github.com/stellar/go/xdr" + "github.com/stellar/wallet-backend/internal/data" + "github.com/stellar/wallet-backend/internal/db" + "github.com/stellar/wallet-backend/internal/db/dbtest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestProcessLedger(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + + dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + m := &IngestManager{ + PaymentModel: &data.PaymentModel{ + DB: dbConnectionPool, + }, + NetworkPassphrase: network.TestNetworkPassphrase, + LedgerCursorName: "last_synced_ledger", + LedgerBackend: nil, + } + + ctx := context.Background() + + // Insert destination account into subscribed addresses + destinationAccount := "GBLI2OE4H3HAW7Z2GXLYZQNQ57XLHJ5OILFPVL33EPA4GDAIQ5F33JGA" + _, err = dbConnectionPool.ExecContext(ctx, "INSERT INTO accounts (stellar_address) VALUES ($1)", destinationAccount) + require.NoError(t, err) + + ledgerMeta := xdr.LedgerCloseMeta{ + V: 1, + V1: &xdr.LedgerCloseMetaV1{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: 123, + ScpValue: xdr.StellarValue{ + CloseTime: xdr.TimePoint(time.Date(2024, 5, 28, 11, 0, 0, 0, time.UTC).Unix()), + }, + LedgerVersion: 10, + }, + }, + TxSet: xdr.GeneralizedTransactionSet{ + V1TxSet: &xdr.TransactionSetV1{ + Phases: []xdr.TransactionPhase{ + { + V0Components: &[]xdr.TxSetComponent{ + { + TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ + Txs: []xdr.TransactionEnvelope{ + { + Type: xdr.EnvelopeTypeEnvelopeTypeTx, + V1: &xdr.TransactionV1Envelope{ + Tx: xdr.Transaction{ + SourceAccount: xdr.MustMuxedAddress("GB3H2CRRTO7W5WF54K53A3MRAFEUISHZ7Y5YGRVGRGHUZESLV5VYYWXI"), + SeqNum: 321, + Memo: xdr.MemoText("memo_test"), + Operations: []xdr.Operation{ + { + SourceAccount: nil, + Body: xdr.OperationBody{ + Type: xdr.OperationTypePayment, + PaymentOp: &xdr.PaymentOp{ + Destination: xdr.MustMuxedAddress(destinationAccount), + Asset: xdr.Asset{ + Type: xdr.AssetTypeAssetTypeCreditAlphanum4, + AlphaNum4: &xdr.AlphaNum4{ + AssetCode: xdr.AssetCode4([]byte("USDC")), + Issuer: xdr.MustMuxedAddress("GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5").ToAccountId(), + }, + }, + Amount: xdr.Int64(50), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + TxProcessing: []xdr.TransactionResultMeta{ + { + Result: xdr.TransactionResultPair{ + TransactionHash: xdr.Hash{}, + }, + TxApplyProcessing: xdr.TransactionMeta{ + V: 3, + }, + }, + }, + }, + } + + // Compute transaction hash and inject into ledger meta + components := ledgerMeta.V1.TxSet.V1TxSet.Phases[0].V0Components + xdrHash, err := network.HashTransactionInEnvelope((*components)[0].TxsMaybeDiscountedFee.Txs[0], m.NetworkPassphrase) + require.NoError(t, err) + ledgerMeta.V1.TxProcessing[0].Result.TransactionHash = xdrHash + + // Run ledger ingestion + err = m.processLedger(ctx, 1, ledgerMeta) + require.NoError(t, err) + + // Assert payment properly persisted to database + var payment data.Payment + query := `SELECT operation_id, operation_type, transaction_id, transaction_hash, from_address, to_address, src_asset_code, src_asset_issuer, src_amount, dest_asset_code, dest_asset_issuer, dest_amount, created_at, memo FROM ingest_payments` + err = dbConnectionPool.GetContext(ctx, &payment, query) + require.NoError(t, err) + assert.Equal(t, data.Payment{ + OperationID: 528280981505, + OperationType: "OperationTypePayment", + TransactionID: 528280981504, + TransactionHash: "c20936e363c85799b31fd321b67aa49ecd88f04fc41297959387e445245080db", + From: "GB3H2CRRTO7W5WF54K53A3MRAFEUISHZ7Y5YGRVGRGHUZESLV5VYYWXI", + To: "GBLI2OE4H3HAW7Z2GXLYZQNQ57XLHJ5OILFPVL33EPA4GDAIQ5F33JGA", + SrcAssetCode: "USDC", + SrcAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5", + SrcAmount: 50, + DestAssetCode: "USDC", + DestAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5", + DestAmount: 50, + CreatedAt: time.Date(2024, 5, 28, 11, 0, 0, 0, time.UTC), + Memo: "memo_test", + }, payment) +} From 1a3876174e676a483c35c70679fe0c6502853ae1 Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Wed, 29 May 2024 14:51:59 -0300 Subject: [PATCH 13/14] rework: memo made optional --- cmd/ingest.go | 2 +- internal/data/payments.go | 2 +- internal/data/payments_test.go | 2 +- internal/services/ingest.go | 6 +++++- internal/services/ingest_test.go | 4 +++- internal/utils/ingestion_utils.go | 29 +++++++++++++++++++------- internal/utils/ingestion_utils_test.go | 10 ++++----- 7 files changed, 38 insertions(+), 17 deletions(-) diff --git a/cmd/ingest.go b/cmd/ingest.go index 7d53fc8..780f880 100644 --- a/cmd/ingest.go +++ b/cmd/ingest.go @@ -62,7 +62,7 @@ func (c *ingestCmd) Command() *cobra.Command { }, { Name: "ledger-cursor-name", - Usage: "Name of last synced ledger cursor, used to keep track of the last ledger ingested by the service. When starting up, ingestion will resume from the ledger number stored in this record. It should be an unique name per container as different containers would overwrite the cursor value of its peers when using the name cursor name.", + Usage: "Name of last synced ledger cursor, used to keep track of the last ledger ingested by the service. When starting up, ingestion will resume from the ledger number stored in this record. It should be an unique name per container as different containers would overwrite the cursor value of its peers when using the same cursor name.", OptType: types.String, ConfigKey: &cfg.LedgerCursorName, Required: true, diff --git a/internal/data/payments.go b/internal/data/payments.go index 603d009..6fbef5b 100644 --- a/internal/data/payments.go +++ b/internal/data/payments.go @@ -27,7 +27,7 @@ type Payment struct { DestAssetIssuer string `db:"dest_asset_issuer"` DestAmount int64 `db:"dest_amount"` CreatedAt time.Time `db:"created_at"` - Memo string `db:"memo"` + Memo *string `db:"memo"` } func (m *PaymentModel) GetLatestLedgerSynced(ctx context.Context, cursorName string) (uint32, error) { diff --git a/internal/data/payments_test.go b/internal/data/payments_test.go index 63e77f5..59672cf 100644 --- a/internal/data/payments_test.go +++ b/internal/data/payments_test.go @@ -44,7 +44,7 @@ func TestAddPayment(t *testing.T) { DestAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5", DestAmount: 500000000, CreatedAt: time.Date(2023, 12, 15, 1, 0, 0, 0, time.UTC), - Memo: "", + Memo: nil, } addPayment := func() { diff --git a/internal/services/ingest.go b/internal/services/ingest.go index 7c23746..44e2b16 100644 --- a/internal/services/ingest.go +++ b/internal/services/ingest.go @@ -139,6 +139,10 @@ func (m *IngestManager) processLedger(ctx context.Context, ledger uint32, ledger txHash := utils.TransactionHash(ledgerMeta, int(tx.Index)) txMemo := utils.Memo(tx.Envelope.Memo(), txHash) + // The memo field is subject to user input, so we sanitize before persisting in the database + if txMemo != nil { + *txMemo = utils.SanitizeUTF8(*txMemo) + } for idx, op := range tx.Envelope.Operations() { opIdx := idx + 1 @@ -150,7 +154,7 @@ func (m *IngestManager) processLedger(ctx context.Context, ledger uint32, ledger TransactionHash: txHash, From: utils.SourceAccount(op, tx), CreatedAt: ledgerCloseTime, - Memo: utils.SanitizeUTF8(txMemo), // Field is subject to user input + Memo: txMemo, } switch op.Body.Type { diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index 327552d..da2cc47 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -123,6 +123,8 @@ func TestProcessLedger(t *testing.T) { query := `SELECT operation_id, operation_type, transaction_id, transaction_hash, from_address, to_address, src_asset_code, src_asset_issuer, src_amount, dest_asset_code, dest_asset_issuer, dest_amount, created_at, memo FROM ingest_payments` err = dbConnectionPool.GetContext(ctx, &payment, query) require.NoError(t, err) + + expectedMemo := "memo_test" assert.Equal(t, data.Payment{ OperationID: 528280981505, OperationType: "OperationTypePayment", @@ -137,6 +139,6 @@ func TestProcessLedger(t *testing.T) { DestAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5", DestAmount: 50, CreatedAt: time.Date(2024, 5, 28, 11, 0, 0, 0, time.UTC), - Memo: "memo_test", + Memo: &expectedMemo, }, payment) } diff --git a/internal/utils/ingestion_utils.go b/internal/utils/ingestion_utils.go index 68cf2b6..9cf4d7a 100644 --- a/internal/utils/ingestion_utils.go +++ b/internal/utils/ingestion_utils.go @@ -26,24 +26,39 @@ func TransactionHash(ledgerMeta xdr.LedgerCloseMeta, txNumber int) string { return ledgerMeta.TransactionHash(txNumber - 1).HexString() } -func Memo(memo xdr.Memo, txHash string) string { +func Memo(memo xdr.Memo, txHash string) *string { memoType := memo.Type switch memoType { case xdr.MemoTypeMemoNone: - return "" + return nil case xdr.MemoTypeMemoText: - return memo.MustText() + if text, ok := memo.GetText(); ok { + return &text + } case xdr.MemoTypeMemoId: - return strconv.FormatUint(uint64(memo.MustId()), 10) + if id, ok := memo.GetId(); ok { + idStr := strconv.FormatUint(uint64(id), 10) + return &idStr + } case xdr.MemoTypeMemoHash: - return memo.MustHash().HexString() + if hash, ok := memo.GetHash(); ok { + hashStr := hash.HexString() + return &hashStr + } case xdr.MemoTypeMemoReturn: - return memo.MustRetHash().HexString() + if retHash, ok := memo.GetRetHash(); ok { + retHashStr := retHash.HexString() + return &retHashStr + } default: // TODO: track in Sentry // sentry.CaptureException(fmt.Errorf("unknown memo type %q for transaction %s", memoType.String(), txHash)) - return "" + return nil } + + // TODO: track in Sentry + // sentry.CaptureException(fmt.Errorf("failed to parse memo for type %q and transaction %s", memoType.String(), txHash)) + return nil } func SourceAccount(op xdr.Operation, tx ingest.LedgerTransaction) string { diff --git a/internal/utils/ingestion_utils_test.go b/internal/utils/ingestion_utils_test.go index d689d98..701b9b9 100644 --- a/internal/utils/ingestion_utils_test.go +++ b/internal/utils/ingestion_utils_test.go @@ -15,7 +15,7 @@ func TestMemo(t *testing.T) { } result := Memo(memo, "") - assert.Equal(t, "", result) + assert.Equal(t, (*string)(nil), result) }) t.Run("type_text", func(t *testing.T) { @@ -26,7 +26,7 @@ func TestMemo(t *testing.T) { } result := Memo(memo, "") - assert.Equal(t, "test", result) + assert.Equal(t, "test", *result) }) t.Run("type_id", func(t *testing.T) { @@ -37,7 +37,7 @@ func TestMemo(t *testing.T) { } result := Memo(memo, "") - assert.Equal(t, "12345", result) + assert.Equal(t, "12345", *result) }) t.Run("type_hash", func(t *testing.T) { @@ -50,7 +50,7 @@ func TestMemo(t *testing.T) { } result := Memo(memo, "") - assert.Equal(t, value.HexString(), result) + assert.Equal(t, value.HexString(), *result) }) t.Run("type_return", func(t *testing.T) { @@ -63,6 +63,6 @@ func TestMemo(t *testing.T) { } result := Memo(memo, "") - assert.Equal(t, value.HexString(), result) + assert.Equal(t, value.HexString(), *result) }) } From 2f04e63be557af37371881104ff903727248d0c3 Mon Sep 17 00:00:00 2001 From: daniel <40469266+daniel-burghardt@users.noreply.github.com> Date: Wed, 29 May 2024 14:52:51 -0300 Subject: [PATCH 14/14] rework: struct name to match db column --- internal/data/payments.go | 6 +++--- internal/data/payments_test.go | 4 ++-- internal/services/ingest.go | 8 ++++---- internal/services/ingest_test.go | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/data/payments.go b/internal/data/payments.go index 6fbef5b..a87bf55 100644 --- a/internal/data/payments.go +++ b/internal/data/payments.go @@ -18,8 +18,8 @@ type Payment struct { OperationType string `db:"operation_type"` TransactionID int64 `db:"transaction_id"` TransactionHash string `db:"transaction_hash"` - From string `db:"from_address"` - To string `db:"to_address"` + FromAddress string `db:"from_address"` + ToAddress string `db:"to_address"` SrcAssetCode string `db:"src_asset_code"` SrcAssetIssuer string `db:"src_asset_issuer"` SrcAmount int64 `db:"src_amount"` @@ -83,7 +83,7 @@ func (m *PaymentModel) AddPayment(ctx context.Context, tx db.Transaction, paymen memo = EXCLUDED.memo ; ` - _, err := tx.ExecContext(ctx, query, payment.OperationID, payment.OperationType, payment.TransactionID, payment.TransactionHash, payment.From, payment.To, payment.SrcAssetCode, payment.SrcAssetIssuer, payment.SrcAmount, + _, err := tx.ExecContext(ctx, query, payment.OperationID, payment.OperationType, payment.TransactionID, payment.TransactionHash, payment.FromAddress, payment.ToAddress, payment.SrcAssetCode, payment.SrcAssetIssuer, payment.SrcAmount, payment.DestAssetCode, payment.DestAssetIssuer, payment.DestAmount, payment.CreatedAt, payment.Memo) if err != nil { return fmt.Errorf("inserting payment: %w", err) diff --git a/internal/data/payments_test.go b/internal/data/payments_test.go index 59672cf..ff866c8 100644 --- a/internal/data/payments_test.go +++ b/internal/data/payments_test.go @@ -35,8 +35,8 @@ func TestAddPayment(t *testing.T) { OperationType: "OperationTypePayment", TransactionID: 2120562792996864, TransactionHash: "a3daffa64dc46db84888b1206dc8014a480042e7fe8b19fd5d05465709f4e887", - From: fromAddress, - To: toAddress, + FromAddress: fromAddress, + ToAddress: toAddress, SrcAssetCode: "USDC", SrcAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5", SrcAmount: 500000000, diff --git a/internal/services/ingest.go b/internal/services/ingest.go index 44e2b16..0c9457a 100644 --- a/internal/services/ingest.go +++ b/internal/services/ingest.go @@ -152,7 +152,7 @@ func (m *IngestManager) processLedger(ctx context.Context, ledger uint32, ledger OperationType: op.Body.Type.String(), TransactionID: utils.TransactionID(int32(ledgerSequence), int32(tx.Index)), TransactionHash: txHash, - From: utils.SourceAccount(op, tx), + FromAddress: utils.SourceAccount(op, tx), CreatedAt: ledgerCloseTime, Memo: txMemo, } @@ -186,7 +186,7 @@ func (m *IngestManager) processLedger(ctx context.Context, ledger uint32, ledger func fillPayment(payment *data.Payment, operation xdr.OperationBody) { paymentOp := operation.MustPaymentOp() - payment.To = paymentOp.Destination.Address() + payment.ToAddress = paymentOp.Destination.Address() payment.SrcAssetCode = utils.AssetCode(paymentOp.Asset) payment.DestAssetCode = payment.SrcAssetCode payment.SrcAssetIssuer = paymentOp.Asset.GetIssuer() @@ -198,7 +198,7 @@ func fillPayment(payment *data.Payment, operation xdr.OperationBody) { func fillPathSend(payment *data.Payment, operation xdr.OperationBody, transaction ingest.LedgerTransaction, operationIdx int) { pathOp := operation.MustPathPaymentStrictSendOp() result := utils.OperationResult(transaction, operationIdx).MustPathPaymentStrictSendResult() - payment.To = pathOp.Destination.Address() + payment.ToAddress = pathOp.Destination.Address() payment.SrcAssetCode = utils.AssetCode(pathOp.SendAsset) payment.DestAssetCode = utils.AssetCode(pathOp.DestAsset) payment.SrcAssetIssuer = pathOp.SendAsset.GetIssuer() @@ -210,7 +210,7 @@ func fillPathSend(payment *data.Payment, operation xdr.OperationBody, transactio func fillPathReceive(payment *data.Payment, operation xdr.OperationBody, transaction ingest.LedgerTransaction, operationIdx int) { pathOp := operation.MustPathPaymentStrictReceiveOp() result := utils.OperationResult(transaction, operationIdx).MustPathPaymentStrictReceiveResult() - payment.To = pathOp.Destination.Address() + payment.ToAddress = pathOp.Destination.Address() payment.SrcAssetCode = utils.AssetCode(pathOp.SendAsset) payment.DestAssetCode = utils.AssetCode(pathOp.DestAsset) payment.SrcAssetIssuer = pathOp.SendAsset.GetIssuer() diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index da2cc47..353e0e0 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -130,8 +130,8 @@ func TestProcessLedger(t *testing.T) { OperationType: "OperationTypePayment", TransactionID: 528280981504, TransactionHash: "c20936e363c85799b31fd321b67aa49ecd88f04fc41297959387e445245080db", - From: "GB3H2CRRTO7W5WF54K53A3MRAFEUISHZ7Y5YGRVGRGHUZESLV5VYYWXI", - To: "GBLI2OE4H3HAW7Z2GXLYZQNQ57XLHJ5OILFPVL33EPA4GDAIQ5F33JGA", + FromAddress: "GB3H2CRRTO7W5WF54K53A3MRAFEUISHZ7Y5YGRVGRGHUZESLV5VYYWXI", + ToAddress: "GBLI2OE4H3HAW7Z2GXLYZQNQ57XLHJ5OILFPVL33EPA4GDAIQ5F33JGA", SrcAssetCode: "USDC", SrcAssetIssuer: "GBBD47IF6LWK7P7MDEVSCWR7DPUWV3NY3DTQEVFL4NAT4AQH3ZLLFLA5", SrcAmount: 50,