diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f1fd33028d..31bd95b8ff 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: [ubuntu-22.04] - go: ["1.21"] + go: ["1.22.1"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -38,7 +38,7 @@ jobs: strategy: matrix: os: [ubuntu-22.04] - go: ["1.20", "1.21"] + go: ["1.21", "1.22"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -56,7 +56,7 @@ jobs: strategy: matrix: os: [ubuntu-22.04] - go: ["1.19", "1.20"] + go: ["1.21", "1.22"] pg: [12] runs-on: ${{ matrix.os }} services: diff --git a/.github/workflows/horizon-release.yml b/.github/workflows/horizon-release.yml index f8dda0ceac..3977ab85d3 100644 --- a/.github/workflows/horizon-release.yml +++ b/.github/workflows/horizon-release.yml @@ -22,7 +22,7 @@ jobs: - uses: ./.github/actions/setup-go with: - go-version: "1.20" + go-version: "1.22" - name: Check dependencies run: ./gomod.sh diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index bf9cae7246..3f95b9e24a 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04, ubuntu-22.04] - go: ["1.20", "1.21"] + go: ["1.21", "1.22"] pg: [12] ingestion-backend: [captive-core, captive-core-remote-storage] protocol-version: [19, 20] @@ -143,7 +143,7 @@ jobs: - name: Build and test the Verify Range Docker image run: | - docker build -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/ + docker build --build-arg="GO_VERSION=$(sed -En 's/^toolchain[[:space:]]+go([[:digit:].]+)$/\1/p' go.mod)" -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/ # Any range should do for basic testing, this range was chosen pretty early in history so that it only takes a few mins to run docker run -e BRANCH=$(git rev-parse HEAD) -e FROM=10000063 -e TO=10000127 stellar/horizon-verify-range diff --git a/clients/horizonclient/main_test.go b/clients/horizonclient/main_test.go index 2149bbbf29..f7b4ba0788 100644 --- a/clients/horizonclient/main_test.go +++ b/clients/horizonclient/main_test.go @@ -3,6 +3,7 @@ package horizonclient import ( "fmt" "net/http" + "strings" "testing" "time" @@ -905,6 +906,25 @@ func TestSubmitTransactionRequest(t *testing.T) { _, err = client.SubmitTransaction(tx) assert.NoError(t, err) + // verify submit parses correctly when result_meta_xdr absent when skip_meta=true + hmock.On( + "POST", + "https://localhost/transactions", + ).Return(func(request *http.Request) (*http.Response, error) { + val := request.FormValue("tx") + assert.Equal(t, val, txXdr) + return httpmock.NewStringResponse(http.StatusOK, strings.Replace(txDetailResponse, "", "", 1)), nil + }) + + hmock.On( + "GET", + "https://localhost/accounts/GACTJ4ZFCDZMD2UFR4R7MZOWYBCF6HBP65YKCUT37MUQFPJLDLJ3N5D2/data/config.memo_required", + ).ReturnString(404, notFoundResponse) + + theTx, err := client.SubmitTransaction(tx) + assert.NoError(t, err) + assert.Empty(t, theTx.ResultMetaXdr) + // memo required - does not submit transaction hmock.On( "GET", @@ -1388,7 +1408,7 @@ func TestTransactionsRequest(t *testing.T) { hmock.On( "GET", "https://localhost/transactions/5131aed266a639a6eb4802a92fba310454e711ded830ed899745b9e777d7110c", - ).ReturnString(200, txDetailResponse) + ).ReturnString(200, strings.Replace(txDetailResponse, "", "result_meta_xdr: AAAAAQAAAAIAAAADAAavdgAAAAAAAAAAtoYrQZHbnPLAFsF4YB88J5VSg0/piQNHm0SL9l0HW1EAAAAXSHbnnAAGr3UAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAABAAavdgAAAAAAAAAAtoYrQZHbnPLAFsF4YB88J5VSg0/piQNHm0SL9l0HW1EAAAAXSHbnnAAGr3UAAAABAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAABAAAABAAAAAMABq9zAAAAAAAAAADMSEvcRKXsaUNna++Hy7gWm/CfqTjEA7xoGypfrFGUHAAAAAUQ/z+cAABeBgAASuQAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAEABq92AAAAAAAAAADMSEvcRKXsaUNna++Hy7gWm/CfqTjEA7xoGypfrFGUHAAAAAcXjracAABeBgAASuQAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAMABq92AAAAAAAAAAC2hitBkduc8sAWwXhgHzwnlVKDT+mJA0ebRIv2XQdbUQAAABdIduecAAavdQAAAAEAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAEABq92AAAAAAAAAAC2hitBkduc8sAWwXhgHzwnlVKDT+mJA0ebRIv2XQdbUQAAABVB53CcAAavdQAAAAEAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAA==", 1)) record, err := client.TransactionDetail(txHash) if assert.NoError(t, err) { @@ -1397,6 +1417,17 @@ func TestTransactionsRequest(t *testing.T) { assert.Equal(t, record.Hash, "5131aed266a639a6eb4802a92fba310454e711ded830ed899745b9e777d7110c") assert.Equal(t, record.Memo, "2A1V6J5703G47XHY") } + + // transaction detail when skip meta enabled and result_meta_xdr is absent + hmock.On( + "GET", + "https://localhost/transactions/5131aed266a639a6eb4802a92fba310454e711ded830ed899745b9e777d7110c", + ).ReturnString(200, strings.Replace(txDetailResponse, "", "", 1)) + + record, err = client.TransactionDetail(txHash) + if assert.NoError(t, err) { + assert.Empty(t, record.ResultMetaXdr) + } } func TestOrderBookRequest(t *testing.T) { @@ -2492,7 +2523,7 @@ var txDetailResponse = `{ "operation_count": 1, "envelope_xdr": "AAAAALaGK0GR25zywBbBeGAfPCeVUoNP6YkDR5tEi/ZdB1tRAAAAZAAGr3UAAAABAAAAAAAAAAEAAAAQMkExVjZKNTcwM0c0N1hIWQAAAAEAAAABAAAAALaGK0GR25zywBbBeGAfPCeVUoNP6YkDR5tEi/ZdB1tRAAAAAQAAAADMSEvcRKXsaUNna++Hy7gWm/CfqTjEA7xoGypfrFGUHAAAAAAAAAACBo93AAAAAAAAAAABXQdbUQAAAECQ5m6ZHsv8/Gd/aRJ2EMLurJMxFynT7KbD51T7gD91Gqp/fzsRHilSGoVSw5ztmtJb2LP7o3bQbiZynQiJPl8C", "result_xdr": "AAAAAAAAAGQAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAA=", - "result_meta_xdr": "AAAAAQAAAAIAAAADAAavdgAAAAAAAAAAtoYrQZHbnPLAFsF4YB88J5VSg0/piQNHm0SL9l0HW1EAAAAXSHbnnAAGr3UAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAABAAavdgAAAAAAAAAAtoYrQZHbnPLAFsF4YB88J5VSg0/piQNHm0SL9l0HW1EAAAAXSHbnnAAGr3UAAAABAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAABAAAABAAAAAMABq9zAAAAAAAAAADMSEvcRKXsaUNna++Hy7gWm/CfqTjEA7xoGypfrFGUHAAAAAUQ/z+cAABeBgAASuQAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAEABq92AAAAAAAAAADMSEvcRKXsaUNna++Hy7gWm/CfqTjEA7xoGypfrFGUHAAAAAcXjracAABeBgAASuQAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAMABq92AAAAAAAAAAC2hitBkduc8sAWwXhgHzwnlVKDT+mJA0ebRIv2XQdbUQAAABdIduecAAavdQAAAAEAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAEABq92AAAAAAAAAAC2hitBkduc8sAWwXhgHzwnlVKDT+mJA0ebRIv2XQdbUQAAABVB53CcAAavdQAAAAEAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAA==", + "result_meta_xdr": "", "fee_meta_xdr": "AAAAAgAAAAMABq91AAAAAAAAAAC2hitBkduc8sAWwXhgHzwnlVKDT+mJA0ebRIv2XQdbUQAAABdIdugAAAavdQAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAEABq92AAAAAAAAAAC2hitBkduc8sAWwXhgHzwnlVKDT+mJA0ebRIv2XQdbUQAAABdIduecAAavdQAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAA==", "memo_type": "text", "signatures": [ diff --git a/exp/services/recoverysigner/docker/Dockerfile b/exp/services/recoverysigner/docker/Dockerfile index 8cd9a72ae6..ff5c14e731 100644 --- a/exp/services/recoverysigner/docker/Dockerfile +++ b/exp/services/recoverysigner/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20-bullseye as build +FROM golang:1.22-bullseye as build ADD . /src/recoverysigner WORKDIR /src/recoverysigner diff --git a/exp/services/webauth/docker/Dockerfile b/exp/services/webauth/docker/Dockerfile index c6bc287d5b..64cff400aa 100644 --- a/exp/services/webauth/docker/Dockerfile +++ b/exp/services/webauth/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20-bullseye as build +FROM golang:1.22-bullseye as build ADD . /src/webauth WORKDIR /src/webauth diff --git a/go.mod b/go.mod index 0e8d89dac2..65d73fa328 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/stellar/go -go 1.20 +go 1.22 + +toolchain go1.22.1 require ( cloud.google.com/go/firestore v1.14.0 // indirect diff --git a/go.sum b/go.sum index ab7ad33230..0fe77e5201 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creachadair/jrpc2 v1.1.0 h1:SgpJf0v1rVCZx68+4APv6dgsTFsIHlpgFD1NlQAWA0A= github.com/creachadair/jrpc2 v1.1.0/go.mod h1:5jN7MKwsm8qvgfTsTzLX3JIfidsAkZ1c8DZSQmp+g38= @@ -121,6 +122,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -129,6 +131,7 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -156,10 +159,13 @@ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfC github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobuffalo/logger v1.0.6 h1:nnZNpxYo0zx+Aj9RfMPBm+x9zAU2OayFh/xrAWi34HU= +github.com/gobuffalo/logger v1.0.6/go.mod h1:J31TBEHR1QLV2683OXTAItYIg8pv2JMHnF/quuAbMjs= github.com/gobuffalo/packd v1.0.2 h1:Yg523YqnOxGIWCp69W12yYBKsoChwI7mtu6ceM9Bwfw= github.com/gobuffalo/packd v1.0.2/go.mod h1:sUc61tDqGMXON80zpKGp92lDb86Km28jfvX7IAyxFT8= github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY= +github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -217,6 +223,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -283,6 +290,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= +github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -293,6 +301,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -309,13 +318,19 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739 h1:ykXz+pRRTibcSjG1yRhpdSHInF8yZY/mfn+Rz2Nd1rE= github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739/go.mod h1:zUx1mhth20V3VKgL5jbd1BSQcW4Fy6Qs4PZvQwRFwzM= github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= +github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= +github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b/go.mod h1:5MWrJXKRQyhQdUCF+vu6U5c4nQpg70vW3eHaU0/AYbU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -332,6 +347,7 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= @@ -351,6 +367,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= +github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -470,6 +487,7 @@ go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znn go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -718,6 +736,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -826,6 +845,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/djherbis/atime.v1 v1.0.0 h1:eMRqB/JrLKocla2PBPKgQYg/p5UG4L6AUAs92aP7F60= gopkg.in/djherbis/atime.v1 v1.0.0/go.mod h1:hQIUStKmJfvf7xdh/wtK84qe+DsTV5LnA9lzxxtPpJ8= gopkg.in/djherbis/stream.v1 v1.3.1 h1:uGfmsOY1qqMjQQphhRBSGLyA9qumJ56exkRu9ASTjCw= @@ -852,6 +872,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/protocols/horizon/README.md b/protocols/horizon/README.md index c6f7ba2654..54b9ed1449 100644 --- a/protocols/horizon/README.md +++ b/protocols/horizon/README.md @@ -17,6 +17,8 @@ For each new version we will only track changes from the previous version. #### Changes +* In ["Transaction"](https://developers.stellar.org/api/horizon/resources/transactions/object), +`result_meta_xdr` field is [now nullable](https://github.com/stellar/go/pull/5228), and will be `null` when Horizon has `SKIP_TXMETA=true` set, otherwise if Horizon is configured with `SKIP_TXMETA=false` which is default, then `result_meta_xdr` will be the same value of base64 encoded xdr. * Operations responses may include a `transaction` field which represents the transaction that created the operation. ### 0.15.0 diff --git a/protocols/horizon/main.go b/protocols/horizon/main.go index 08da47b7b4..bdec98ba0b 100644 --- a/protocols/horizon/main.go +++ b/protocols/horizon/main.go @@ -518,7 +518,7 @@ type Transaction struct { OperationCount int32 `json:"operation_count"` EnvelopeXdr string `json:"envelope_xdr"` ResultXdr string `json:"result_xdr"` - ResultMetaXdr string `json:"result_meta_xdr"` + ResultMetaXdr string `json:"result_meta_xdr,omitempty"` FeeMetaXdr string `json:"fee_meta_xdr"` MemoType string `json:"memo_type"` MemoBytes string `json:"memo_bytes,omitempty"` diff --git a/services/friendbot/docker/Dockerfile b/services/friendbot/docker/Dockerfile index dc1c74b93f..764fa5e276 100644 --- a/services/friendbot/docker/Dockerfile +++ b/services/friendbot/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20-bullseye as build +FROM golang:1.22-bullseye as build ADD . /src/friendbot WORKDIR /src/friendbot diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 7bf15774f1..003bab1d5a 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -5,12 +5,26 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +## 2.29.0 + ### Added - New `db_error_total` metrics key with labels `ctx_error`, `db_error`, and `db_error_extra` ([5225](https://github.com/stellar/go/pull/5225)). +- Bumped go version to the latest (1.22.1) ([5232](https://github.com/stellar/go/pull/5232)) +- Add metrics for ingestion loaders ([5209](https://github.com/stellar/go/pull/5209)). +- Add metrics for http api requests in flight and requests received ([5240](https://github.com/stellar/go/pull/5240)). +- Add `MAX_CONCURRENT_REQUESTS`, defaults to 1000, limits the number of horizon api requests in flight ([5244](https://github.com/stellar/go/pull/5244)) ### Fixed - History archive access is more effective when you pass list of URLs to Horizon: they will now be accessed in a round-robin fashion, use alternative archives on errors, and intelligently back off ([5224](https://github.com/stellar/go/pull/5224)) +- Remove captive core info request error logs ([5145](https://github.com/stellar/go/pull/5145)) +- Removed duplicate "Processed Ledger" log statement during resume state ([5152](https://github.com/stellar/go/pull/5152)) +- Fixed incorrect duration for ingestion processor metric ([5216](https://github.com/stellar/go/pull/5216)) +- Fixed sql performance on account transactions query ([5229](https://github.com/stellar/go/pull/5229)) +- Fix bug in claimable balance change processor ([5246](https://github.com/stellar/go/pull/5246)) +- Delay canceling queries from client side when there's a statement / transaction timeout configured in postgres ([5223](https://github.com/stellar/go/pull/5223)) +### Breaking Changes +- The Horizon API Transaction resource field in json `result_meta_xdr` is now optional and Horizon API will not emit the field when Horizon has been configured with `SKIP_TXMETA=true`, effectively null, otherwise if Horizon is configured with `SKIP_TXMETA=false` which is default, then the API Transaction field `result_meta_xdr` will remain present and populated with base64 encoded xdr [5228](https://github.com/stellar/go/pull/5228). ## 2.28.3 diff --git a/services/horizon/Makefile b/services/horizon/Makefile index 9c5a3a8ddf..0789453373 100644 --- a/services/horizon/Makefile +++ b/services/horizon/Makefile @@ -11,7 +11,7 @@ binary-build: --pull always \ --env CGO_ENABLED=0 \ --env GOFLAGS="-ldflags=-X=github.com/stellar/go/support/app.version=$(VERSION_STRING)" \ - golang:1.20-bullseye \ + golang:1.22-bullseye \ /bin/bash -c '\ git config --global --add safe.directory /go/src/github.com/stellar/go && \ cd /go/src/github.com/stellar/go && \ diff --git a/services/horizon/docker/Dockerfile.dev b/services/horizon/docker/Dockerfile.dev index 1d1be8d688..5cef8d89d1 100644 --- a/services/horizon/docker/Dockerfile.dev +++ b/services/horizon/docker/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.20-bullseye AS builder +FROM golang:1.22-bullseye AS builder ARG VERSION="devel" WORKDIR /go/src/github.com/stellar/go diff --git a/services/horizon/docker/verify-range/Dockerfile b/services/horizon/docker/verify-range/Dockerfile index 0143dd2cfa..6323870f38 100644 --- a/services/horizon/docker/verify-range/Dockerfile +++ b/services/horizon/docker/verify-range/Dockerfile @@ -1,5 +1,6 @@ FROM ubuntu:22.04 +ARG GO_VERSION ARG STELLAR_CORE_VERSION ENV STELLAR_CORE_VERSION=${STELLAR_CORE_VERSION:-*} # to remove tzdata interactive flow diff --git a/services/horizon/docker/verify-range/dependencies b/services/horizon/docker/verify-range/dependencies index 910ee9cf21..3eacede44b 100644 --- a/services/horizon/docker/verify-range/dependencies +++ b/services/horizon/docker/verify-range/dependencies @@ -19,7 +19,6 @@ cd stellar-go git config --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pull/*" git fetch --force --quiet origin -GO_VERSION=$(sed -En 's/^go[[:space:]]+([[:digit:].]+)$/\1/p' go.mod) wget -q https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz rm -f go${GO_VERSION}.linux-amd64.tar.gz diff --git a/services/horizon/internal/actions/effects.go b/services/horizon/internal/actions/effects.go index a141067d25..cc7406d7bc 100644 --- a/services/horizon/internal/actions/effects.go +++ b/services/horizon/internal/actions/effects.go @@ -95,25 +95,20 @@ func (handler GetEffectsHandler) GetResourcePage(w HeaderWriter, r *http.Request } func loadEffectRecords(ctx context.Context, hq *history.Q, qp EffectsQuery, pq db2.PageQuery) ([]history.Effect, error) { - effects := hq.Effects() - switch { case qp.AccountID != "": - effects.ForAccount(ctx, qp.AccountID) + return hq.EffectsForAccount(ctx, qp.AccountID, pq) case qp.LiquidityPoolID != "": - effects.ForLiquidityPool(ctx, pq, qp.LiquidityPoolID) + return hq.EffectsForLiquidityPool(ctx, qp.LiquidityPoolID, pq) case qp.OperationID > 0: - effects.ForOperation(int64(qp.OperationID)) + return hq.EffectsForOperation(ctx, int64(qp.OperationID), pq) case qp.LedgerID > 0: - effects.ForLedger(ctx, int32(qp.LedgerID)) + return hq.EffectsForLedger(ctx, int32(qp.LedgerID), pq) case qp.TxHash != "": - effects.ForTransaction(ctx, qp.TxHash) + return hq.EffectsForTransaction(ctx, qp.TxHash, pq) + default: + return hq.Effects(ctx, pq) } - - var result []history.Effect - err := effects.Page(pq).Select(ctx, &result) - - return result, err } func loadEffectLedgers(ctx context.Context, hq *history.Q, effects []history.Effect) (map[int32]history.Ledger, error) { diff --git a/services/horizon/internal/actions/operation.go b/services/horizon/internal/actions/operation.go index 670f592b57..f59191aee0 100644 --- a/services/horizon/internal/actions/operation.go +++ b/services/horizon/internal/actions/operation.go @@ -65,6 +65,7 @@ func (qp OperationsQuery) Validate() error { type GetOperationsHandler struct { LedgerState *ledger.State OnlyPayments bool + SkipTxMeta bool } // GetResourcePage returns a page of operations. @@ -126,12 +127,13 @@ func (handler GetOperationsHandler) GetResourcePage(w HeaderWriter, r *http.Requ return nil, err } - return buildOperationsPage(ctx, historyQ, ops, txs, qp.IncludeTransactions()) + return buildOperationsPage(ctx, historyQ, ops, txs, qp.IncludeTransactions(), handler.SkipTxMeta) } // GetOperationByIDHandler is the action handler for all end-points returning a list of operations. type GetOperationByIDHandler struct { LedgerState *ledger.State + SkipTxMeta bool } // OperationQuery query struct for operation/id end-point @@ -182,10 +184,11 @@ func (handler GetOperationByIDHandler) GetResource(w HeaderWriter, r *http.Reque op.TransactionHash, tx, ledger, + handler.SkipTxMeta, ) } -func buildOperationsPage(ctx context.Context, historyQ *history.Q, operations []history.Operation, transactions []history.Transaction, includeTransactions bool) ([]hal.Pageable, error) { +func buildOperationsPage(ctx context.Context, historyQ *history.Q, operations []history.Operation, transactions []history.Transaction, includeTransactions bool, skipTxMeta bool) ([]hal.Pageable, error) { ledgerCache := history.LedgerCache{} for _, record := range operations { ledgerCache.Queue(record.LedgerSequence()) @@ -216,6 +219,7 @@ func buildOperationsPage(ctx context.Context, historyQ *history.Q, operations [] operationRecord.TransactionHash, transactionRecord, ledger, + skipTxMeta, ) if err != nil { return nil, err diff --git a/services/horizon/internal/actions/submit_transaction.go b/services/horizon/internal/actions/submit_transaction.go index b877f75a7b..314caf32a5 100644 --- a/services/horizon/internal/actions/submit_transaction.go +++ b/services/horizon/internal/actions/submit_transaction.go @@ -27,6 +27,7 @@ type SubmitTransactionHandler struct { NetworkPassphrase string DisableTxSub bool CoreStateGetter + SkipTxMeta bool } type envelopeInfo struct { @@ -84,6 +85,7 @@ func (handler SubmitTransactionHandler) response(r *http.Request, info envelopeI info.hash, &resource, result.Transaction, + handler.SkipTxMeta, ) return resource, err } diff --git a/services/horizon/internal/actions/transaction.go b/services/horizon/internal/actions/transaction.go index f823ad9acb..6903d5db2f 100644 --- a/services/horizon/internal/actions/transaction.go +++ b/services/horizon/internal/actions/transaction.go @@ -23,6 +23,7 @@ type TransactionQuery struct { // GetTransactionByHashHandler is the action handler for the end-point returning a transaction. type GetTransactionByHashHandler struct { + SkipTxMeta bool } // GetResource returns a transaction page. @@ -49,7 +50,7 @@ func (handler GetTransactionByHashHandler) GetResource(w HeaderWriter, r *http.R return resource, errors.Wrap(err, "loading transaction record") } - if err = resourceadapter.PopulateTransaction(ctx, qp.TransactionHash, &resource, record); err != nil { + if err = resourceadapter.PopulateTransaction(ctx, qp.TransactionHash, &resource, record, handler.SkipTxMeta); err != nil { return resource, errors.Wrap(err, "could not populate transaction") } return resource, nil @@ -90,6 +91,7 @@ func (qp TransactionsQuery) Validate() error { // GetTransactionsHandler is the action handler for all end-points returning a list of transactions. type GetTransactionsHandler struct { LedgerState *ledger.State + SkipTxMeta bool } // GetResourcePage returns a page of transactions. @@ -126,7 +128,7 @@ func (handler GetTransactionsHandler) GetResourcePage(w HeaderWriter, r *http.Re for _, record := range records { var res horizon.Transaction - err = resourceadapter.PopulateTransaction(ctx, record.TransactionHash, &res, record) + err = resourceadapter.PopulateTransaction(ctx, record.TransactionHash, &res, record, handler.SkipTxMeta) if err != nil { return nil, errors.Wrap(err, "could not populate transaction") } diff --git a/services/horizon/internal/app.go b/services/horizon/internal/app.go index b1cd7a1c85..8fb86b5f54 100644 --- a/services/horizon/internal/app.go +++ b/services/horizon/internal/app.go @@ -532,6 +532,8 @@ func (a *App) init() error { SSEUpdateFrequency: a.config.SSEUpdateFrequency, StaleThreshold: a.config.StaleThreshold, ConnectionTimeout: a.config.ConnectionTimeout, + ClientQueryTimeout: a.config.ClientQueryTimeout, + MaxConcurrentRequests: a.config.MaxConcurrentRequests, MaxHTTPRequestSize: a.config.MaxHTTPRequestSize, NetworkPassphrase: a.config.NetworkPassphrase, MaxPathLength: a.config.MaxPathLength, @@ -552,6 +554,7 @@ func (a *App) init() error { }, cache: newHealthCache(healthCacheTTL), }, + SkipTxMeta: a.config.SkipTxmeta, } if a.primaryHistoryQ != nil { diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go index f1f8ac078d..94b6f5514b 100644 --- a/services/horizon/internal/config.go +++ b/services/horizon/internal/config.go @@ -38,12 +38,14 @@ type Config struct { SSEUpdateFrequency time.Duration ConnectionTimeout time.Duration + ClientQueryTimeout time.Duration // MaxHTTPRequestSize is the maximum allowed request payload size - MaxHTTPRequestSize uint - RateQuota *throttled.RateQuota - FriendbotURL *url.URL - LogLevel logrus.Level - LogFile string + MaxHTTPRequestSize uint + RateQuota *throttled.RateQuota + MaxConcurrentRequests uint + FriendbotURL *url.URL + LogLevel logrus.Level + LogFile string // MaxPathLength is the maximum length of the path returned by `/paths` endpoint. MaxPathLength uint diff --git a/services/horizon/internal/db2/history/asset_loader.go b/services/horizon/internal/db2/history/asset_loader.go index bdf40fb843..fe17dc17be 100644 --- a/services/horizon/internal/db2/history/asset_loader.go +++ b/services/horizon/internal/db2/history/asset_loader.go @@ -7,8 +7,6 @@ import ( "sort" "strings" - sq "github.com/Masterminds/squirrel" - "github.com/stellar/go/support/collections/set" "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" @@ -108,13 +106,17 @@ func (a *AssetLoader) lookupKeys(ctx context.Context, q *Q, keys []AssetKey) err for i := 0; i < len(keys); i += loaderLookupBatchSize { end := ordered.Min(len(keys), i+loaderLookupBatchSize) subset := keys[i:end] - keyStrings := make([]string, 0, len(subset)) + args := make([]interface{}, 0, 3*len(subset)) + placeHolders := make([]string, 0, len(subset)) for _, key := range subset { - keyStrings = append(keyStrings, key.Type+"/"+key.Code+"/"+key.Issuer) + args = append(args, key.Code, key.Type, key.Issuer) + placeHolders = append(placeHolders, "(?, ?, ?)") } - err := q.Select(ctx, &rows, sq.Select("*").From("history_assets").Where(sq.Eq{ - "concat(asset_type, '/', asset_code, '/', asset_issuer)": keyStrings, - })) + rawSQL := fmt.Sprintf( + "SELECT * FROM history_assets WHERE (asset_code, asset_type, asset_issuer) in (%s)", + strings.Join(placeHolders, ", "), + ) + err := q.SelectRaw(ctx, &rows, rawSQL, args...) if err != nil { return errors.Wrap(err, "could not select assets") } diff --git a/services/horizon/internal/db2/history/claimable_balances.go b/services/horizon/internal/db2/history/claimable_balances.go index c198ee162d..abdf4ed758 100644 --- a/services/horizon/internal/db2/history/claimable_balances.go +++ b/services/horizon/internal/db2/history/claimable_balances.go @@ -140,6 +140,7 @@ type Claimant struct { // QClaimableBalances defines claimable-balance-related related queries. type QClaimableBalances interface { + UpsertClaimableBalances(ctx context.Context, cb []ClaimableBalance) error RemoveClaimableBalances(ctx context.Context, ids []string) (int64, error) RemoveClaimableBalanceClaimants(ctx context.Context, ids []string) (int64, error) GetClaimableBalancesByID(ctx context.Context, ids []string) ([]ClaimableBalance, error) @@ -185,6 +186,66 @@ func (q *Q) GetClaimantsByClaimableBalances(ctx context.Context, ids []string) ( return claimantsMap, err } +// UpsertClaimableBalances upserts a batch of claimable balances in the claimable_balances table. +// It also upserts the corresponding claimants in the claimable_balance_claimants table. +func (q *Q) UpsertClaimableBalances(ctx context.Context, cbs []ClaimableBalance) error { + if err := q.upsertCBs(ctx, cbs); err != nil { + return errors.Wrap(err, "could not upsert claimable balances") + } + + if err := q.upsertCBClaimants(ctx, cbs); err != nil { + return errors.Wrap(err, "could not upsert claimable balance claimants") + } + + return nil +} + +func (q *Q) upsertCBClaimants(ctx context.Context, cbs []ClaimableBalance) error { + var id, lastModifiedLedger, destination []interface{} + + for _, cb := range cbs { + for _, claimant := range cb.Claimants { + id = append(id, cb.BalanceID) + lastModifiedLedger = append(lastModifiedLedger, cb.LastModifiedLedger) + destination = append(destination, claimant.Destination) + } + } + + upsertFields := []upsertField{ + {"id", "text", id}, + {"destination", "text", destination}, + {"last_modified_ledger", "integer", lastModifiedLedger}, + } + + return q.upsertRows(ctx, "claimable_balance_claimants", "id, destination", upsertFields) +} + +func (q *Q) upsertCBs(ctx context.Context, cbs []ClaimableBalance) error { + var id, claimants, asset, amount, sponsor, lastModifiedLedger, flags []interface{} + + for _, cb := range cbs { + id = append(id, cb.BalanceID) + claimants = append(claimants, cb.Claimants) + asset = append(asset, cb.Asset) + amount = append(amount, cb.Amount) + sponsor = append(sponsor, cb.Sponsor) + lastModifiedLedger = append(lastModifiedLedger, cb.LastModifiedLedger) + flags = append(flags, cb.Flags) + } + + upsertFields := []upsertField{ + {"id", "text", id}, + {"claimants", "jsonb", claimants}, + {"asset", "text", asset}, + {"amount", "bigint", amount}, + {"sponsor", "text", sponsor}, + {"last_modified_ledger", "integer", lastModifiedLedger}, + {"flags", "int", flags}, + } + + return q.upsertRows(ctx, "claimable_balances", "id", upsertFields) +} + // RemoveClaimableBalances deletes claimable balances table. // Returns number of rows affected and error. func (q *Q) RemoveClaimableBalances(ctx context.Context, ids []string) (int64, error) { diff --git a/services/horizon/internal/db2/history/claimable_balances_test.go b/services/horizon/internal/db2/history/claimable_balances_test.go index 2e6d621945..1ffe442244 100644 --- a/services/horizon/internal/db2/history/claimable_balances_test.go +++ b/services/horizon/internal/db2/history/claimable_balances_test.go @@ -588,6 +588,85 @@ func TestFindClaimableBalancesByDestinationWithLimit(t *testing.T) { }) } +func TestUpdateClaimableBalance(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + q := &Q{tt.HorizonSession()} + + accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" + lastModifiedLedgerSeq := xdr.Uint32(123) + asset := xdr.MustNewCreditAsset("USD", accountID) + balanceID := xdr.ClaimableBalanceId{ + Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0, + V0: &xdr.Hash{1, 2, 3}, + } + id, err := xdr.MarshalHex(balanceID) + tt.Assert.NoError(err) + cBalance := ClaimableBalance{ + BalanceID: id, + Claimants: []Claimant{ + { + Destination: accountID, + Predicate: xdr.ClaimPredicate{ + Type: xdr.ClaimPredicateTypeClaimPredicateUnconditional, + }, + }, + }, + Asset: asset, + LastModifiedLedger: 123, + Amount: 10, + } + + err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance}) + tt.Assert.NoError(err) + + cBalancesClaimants, err := q.GetClaimantsByClaimableBalances(tt.Ctx, []string{cBalance.BalanceID}) + tt.Assert.NoError(err) + tt.Assert.Len(cBalancesClaimants[cBalance.BalanceID], 1) + tt.Assert.Equal(ClaimableBalanceClaimant{ + BalanceID: cBalance.BalanceID, + Destination: accountID, + LastModifiedLedger: cBalance.LastModifiedLedger, + }, cBalancesClaimants[cBalance.BalanceID][0]) + + // add sponsor + cBalance2 := ClaimableBalance{ + BalanceID: id, + Claimants: []Claimant{ + { + Destination: accountID, + Predicate: xdr.ClaimPredicate{ + Type: xdr.ClaimPredicateTypeClaimPredicateUnconditional, + }, + }, + }, + Asset: asset, + LastModifiedLedger: 123 + 1, + Amount: 10, + Sponsor: null.StringFrom("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), + } + + err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance2}) + tt.Assert.NoError(err) + + cbs := []ClaimableBalance{} + err = q.Select(tt.Ctx, &cbs, selectClaimableBalances) + tt.Assert.NoError(err) + tt.Assert.Len(cbs, 1) + tt.Assert.Equal("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML", cbs[0].Sponsor.String) + tt.Assert.Equal(uint32(lastModifiedLedgerSeq+1), cbs[0].LastModifiedLedger) + + cBalancesClaimants, err = q.GetClaimantsByClaimableBalances(tt.Ctx, []string{cBalance2.BalanceID}) + tt.Assert.NoError(err) + tt.Assert.Len(cBalancesClaimants[cBalance2.BalanceID], 1) + tt.Assert.Equal(ClaimableBalanceClaimant{ + BalanceID: cBalance2.BalanceID, + Destination: accountID, + LastModifiedLedger: cBalance2.LastModifiedLedger, + }, cBalancesClaimants[cBalance2.BalanceID][0]) +} + func TestFindClaimableBalance(t *testing.T) { tt := test.Start(t) defer tt.Finish() diff --git a/services/horizon/internal/db2/history/effect.go b/services/horizon/internal/db2/history/effect.go index 13a9c52519..bdf1e2dfb0 100644 --- a/services/horizon/internal/db2/history/effect.go +++ b/services/horizon/internal/db2/history/effect.go @@ -69,75 +69,83 @@ func (r *Effect) PagingToken() string { return fmt.Sprintf("%d-%d", r.HistoryOperationID, r.Order) } -// Effects provides a helper to filter rows from the `history_effects` -// table with pre-defined filters. See `TransactionsQ` methods for the -// available filters. -func (q *Q) Effects() *EffectsQ { - return &EffectsQ{ - parent: q, - sql: selectEffect, +// Effects returns a page of effects without any filters besides the cursor +func (q *Q) Effects(ctx context.Context, page db2.PageQuery) ([]Effect, error) { + op, idx, err := parseEffectsCursor(page) + if err != nil { + return nil, err + } + + var rows []Effect + query := selectEffect + // we do not use selectEffectsPage() because we have found the + // query below to be more efficient when there are no other constraints + // such as filtering by account / ledger / transaction / etc + switch page.Order { + case "asc": + query = query. + Where("(heff.history_operation_id, heff.order) > (?, ?)", op, idx). + OrderBy("heff.history_operation_id asc, heff.order asc") + case "desc": + query = query. + Where("(heff.history_operation_id, heff.order) < (?, ?)", op, idx). + OrderBy("heff.history_operation_id desc, heff.order desc") } + + query = query.Limit(page.Limit) + + if err = q.Select(ctx, &rows, query); err != nil { + return nil, err + } + return rows, nil } -// ForAccount filters the operations collection to a specific account -func (q *EffectsQ) ForAccount(ctx context.Context, aid string) *EffectsQ { +// EffectsForAccount returns a page of effects for a given account +func (q *Q) EffectsForAccount(ctx context.Context, aid string, page db2.PageQuery) ([]Effect, error) { var account Account - q.Err = q.parent.AccountByAddress(ctx, &account, aid) - if q.Err != nil { - return q + if err := q.AccountByAddress(ctx, &account, aid); err != nil { + return nil, err } - q.sql = q.sql.Where("heff.history_account_id = ?", account.ID) - - return q + query := selectEffect.Where("heff.history_account_id = ?", account.ID) + return q.selectEffectsPage(ctx, query, page) } -// ForLedger filters the query to only effects in a specific ledger, -// specified by its sequence. -func (q *EffectsQ) ForLedger(ctx context.Context, seq int32) *EffectsQ { +// EffectsForLedger returns a page of effects for a given ledger sequence +func (q *Q) EffectsForLedger(ctx context.Context, seq int32, page db2.PageQuery) ([]Effect, error) { var ledger Ledger - q.Err = q.parent.LedgerBySequence(ctx, &ledger, seq) - if q.Err != nil { - return q + if err := q.LedgerBySequence(ctx, &ledger, seq); err != nil { + return nil, err } start := toid.ID{LedgerSequence: seq} end := toid.ID{LedgerSequence: seq + 1} - q.sql = q.sql.Where( + query := selectEffect.Where( "heff.history_operation_id >= ? AND heff.history_operation_id < ?", start.ToInt64(), end.ToInt64(), ) - - return q + return q.selectEffectsPage(ctx, query, page) } -// ForOperation filters the query to only effects in a specific operation, -// specified by its id. -func (q *EffectsQ) ForOperation(id int64) *EffectsQ { +// EffectsForOperation returns a page of effects for a given operation id. +func (q *Q) EffectsForOperation(ctx context.Context, id int64, page db2.PageQuery) ([]Effect, error) { start := toid.Parse(id) end := start end.IncOperationOrder() - q.sql = q.sql.Where( + query := selectEffect.Where( "heff.history_operation_id >= ? AND heff.history_operation_id < ?", start.ToInt64(), end.ToInt64(), ) - - return q + return q.selectEffectsPage(ctx, query, page) } -// ForLiquidityPool filters the query to only effects in a specific liquidity pool, -// specified by its id. -func (q *EffectsQ) ForLiquidityPool(ctx context.Context, page db2.PageQuery, id string) *EffectsQ { - if q.Err != nil { - return q - } - +// EffectsForLiquidityPool returns a page of effects for a given liquidity pool. +func (q *Q) EffectsForLiquidityPool(ctx context.Context, id string, page db2.PageQuery) ([]Effect, error) { op, _, err := page.CursorInt64Pair(db2.DefaultPairSep) if err != nil { - q.Err = err - return q + return nil, err } query := `SELECT holp.history_operation_id @@ -150,59 +158,62 @@ func (q *EffectsQ) ForLiquidityPool(ctx context.Context, page db2.PageQuery, id case "desc": query += "AND holp.history_operation_id <= ? ORDER BY holp.history_operation_id desc LIMIT ?" default: - q.Err = errors.Errorf("invalid paging order: %s", page.Order) - return q + return nil, errors.Errorf("invalid paging order: %s", page.Order) } var liquidityPoolOperationIDs []int64 - err = q.parent.SelectRaw(ctx, &liquidityPoolOperationIDs, query, id, op, page.Limit) + err = q.SelectRaw(ctx, &liquidityPoolOperationIDs, query, id, op, page.Limit) if err != nil { - q.Err = err - return q + return nil, err } - q.sql = q.sql.Where(map[string]interface{}{ - "heff.history_operation_id": liquidityPoolOperationIDs, - }) - return q + return q.selectEffectsPage( + ctx, + selectEffect.Where(map[string]interface{}{ + "heff.history_operation_id": liquidityPoolOperationIDs, + }), + page, + ) } -// ForTransaction filters the query to only effects in a specific -// transaction, specified by the transactions's hex-encoded hash. -func (q *EffectsQ) ForTransaction(ctx context.Context, hash string) *EffectsQ { +// EffectsForTransaction returns a page of effects for a given transaction +func (q *Q) EffectsForTransaction(ctx context.Context, hash string, page db2.PageQuery) ([]Effect, error) { var tx Transaction - q.Err = q.parent.TransactionByHash(ctx, &tx, hash) - if q.Err != nil { - return q + if err := q.TransactionByHash(ctx, &tx, hash); err != nil { + return nil, err } start := toid.Parse(tx.ID) end := start end.TransactionOrder++ - q.sql = q.sql.Where( - "heff.history_operation_id >= ? AND heff.history_operation_id < ?", - start.ToInt64(), - end.ToInt64(), - ) - return q + return q.selectEffectsPage( + ctx, + selectEffect.Where("heff.history_operation_id >= ? AND heff.history_operation_id < ?", + start.ToInt64(), + end.ToInt64(), + ), + page, + ) } -// Page specifies the paging constraints for the query being built by `q`. -func (q *EffectsQ) Page(page db2.PageQuery) *EffectsQ { - if q.Err != nil { - return q - } - +func parseEffectsCursor(page db2.PageQuery) (int64, int64, error) { op, idx, err := page.CursorInt64Pair(db2.DefaultPairSep) if err != nil { - q.Err = err - return q + return 0, 0, err } if idx > math.MaxInt32 { idx = math.MaxInt32 } + return op, idx, nil +} + +func (q *Q) selectEffectsPage(ctx context.Context, query sq.SelectBuilder, page db2.PageQuery) ([]Effect, error) { + op, idx, err := parseEffectsCursor(page) + if err != nil { + return nil, err + } // NOTE: Remember to test the queries below with EXPLAIN / EXPLAIN ANALYZE // before changing them. @@ -210,7 +221,7 @@ func (q *EffectsQ) Page(page db2.PageQuery) *EffectsQ { // DB will perform a full table scan. switch page.Order { case "asc": - q.sql = q.sql. + query = query. Where(`( heff.history_operation_id >= ? AND ( @@ -219,7 +230,7 @@ func (q *EffectsQ) Page(page db2.PageQuery) *EffectsQ { ))`, op, op, op, idx). OrderBy("heff.history_operation_id asc, heff.order asc") case "desc": - q.sql = q.sql. + query = query. Where(`( heff.history_operation_id <= ? AND ( @@ -229,18 +240,14 @@ func (q *EffectsQ) Page(page db2.PageQuery) *EffectsQ { OrderBy("heff.history_operation_id desc, heff.order desc") } - q.sql = q.sql.Limit(page.Limit) - return q -} + query = query.Limit(page.Limit) -// Select loads the results of the query specified by `q` into `dest`. -func (q *EffectsQ) Select(ctx context.Context, dest interface{}) error { - if q.Err != nil { - return q.Err + var rows []Effect + if err = q.Select(ctx, &rows, query); err != nil { + return nil, err } - q.Err = q.parent.Select(ctx, dest, q.sql) - return q.Err + return rows, nil } // QEffects defines history_effects related queries. diff --git a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go index dc02148a7d..e1ac998953 100644 --- a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go +++ b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go @@ -6,6 +6,7 @@ import ( "github.com/guregu/null" + "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/toid" ) @@ -42,8 +43,12 @@ func TestAddEffect(t *testing.T) { tt.Assert.NoError(builder.Exec(tt.Ctx, q)) tt.Assert.NoError(q.Commit()) - effects := []Effect{} - tt.Assert.NoError(q.Effects().Select(tt.Ctx, &effects)) + effects, err := q.Effects(tt.Ctx, db2.PageQuery{ + Cursor: "0-0", + Order: "asc", + Limit: 200, + }) + tt.Require.NoError(err) tt.Assert.Len(effects, 1) effect := effects[0] diff --git a/services/horizon/internal/db2/history/effect_test.go b/services/horizon/internal/db2/history/effect_test.go index 498d5e92df..19af0ceff8 100644 --- a/services/horizon/internal/db2/history/effect_test.go +++ b/services/horizon/internal/db2/history/effect_test.go @@ -57,11 +57,11 @@ func TestEffectsForLiquidityPool(t *testing.T) { tt.Assert.NoError(q.Commit()) var result []Effect - err = q.Effects().ForLiquidityPool(tt.Ctx, db2.PageQuery{ + result, err = q.EffectsForLiquidityPool(tt.Ctx, liquidityPoolID, db2.PageQuery{ Cursor: "0-0", Order: "asc", Limit: 10, - }, liquidityPoolID).Select(tt.Ctx, &result) + }) tt.Assert.NoError(err) tt.Assert.Len(result, 1) @@ -156,8 +156,12 @@ func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) { tt.Require.NoError(builder.Exec(tt.Ctx, q)) tt.Assert.NoError(q.Commit()) - var results []Effect - tt.Require.NoError(q.Effects().Select(tt.Ctx, &results)) + results, err := q.Effects(tt.Ctx, db2.PageQuery{ + Cursor: "0-0", + Order: "asc", + Limit: 200, + }) + tt.Require.NoError(err) tt.Require.Len(results, len(tests)) for i, test := range tests { diff --git a/services/horizon/internal/db2/history/liquidity_pools.go b/services/horizon/internal/db2/history/liquidity_pools.go index 46e6ba59d3..3259163a89 100644 --- a/services/horizon/internal/db2/history/liquidity_pools.go +++ b/services/horizon/internal/db2/history/liquidity_pools.go @@ -9,8 +9,9 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/guregu/null" - "github.com/jmoiron/sqlx" + "github.com/stellar/go/services/horizon/internal/db2" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) @@ -188,7 +189,7 @@ func (q *Q) GetLiquidityPools(ctx context.Context, query LiquidityPoolsQuery) ([ } func (q *Q) StreamAllLiquidityPools(ctx context.Context, callback func(LiquidityPool) error) error { - var rows *sqlx.Rows + var rows *db.Rows var err error if rows, err = q.Query(ctx, selectLiquidityPools.Where("deleted = ?", false)); err != nil { diff --git a/services/horizon/internal/db2/history/liquidity_pools_test.go b/services/horizon/internal/db2/history/liquidity_pools_test.go index fd268d2518..2488945168 100644 --- a/services/horizon/internal/db2/history/liquidity_pools_test.go +++ b/services/horizon/internal/db2/history/liquidity_pools_test.go @@ -112,6 +112,7 @@ func TestStreamAllLiquidity(t *testing.T) { pools = append(pools, pool) return nil }) + tt.Assert.NoError(err) sort.Slice(pools, func(i, j int) bool { return pools[i].PoolID < pools[j].PoolID }) diff --git a/services/horizon/internal/db2/history/main.go b/services/horizon/internal/db2/history/main.go index 41a4cb0068..d9c5ea7557 100644 --- a/services/horizon/internal/db2/history/main.go +++ b/services/horizon/internal/db2/history/main.go @@ -632,14 +632,6 @@ type SequenceBumped struct { NewSeq int64 `json:"new_seq"` } -// EffectsQ is a helper struct to aid in configuring queries that loads -// slices of Ledger structs. -type EffectsQ struct { - Err error - parent *Q - sql sq.SelectBuilder -} - // EffectType is the numeric type for an effect, used as the `type` field in the // `history_effects` table. type EffectType int diff --git a/services/horizon/internal/db2/history/mock_q_claimable_balances.go b/services/horizon/internal/db2/history/mock_q_claimable_balances.go index 64b65cf1a3..6a3adffac1 100644 --- a/services/horizon/internal/db2/history/mock_q_claimable_balances.go +++ b/services/horizon/internal/db2/history/mock_q_claimable_balances.go @@ -21,6 +21,11 @@ func (m *MockQClaimableBalances) GetClaimableBalancesByID(ctx context.Context, i return a.Get(0).([]ClaimableBalance), a.Error(1) } +func (m *MockQClaimableBalances) UpsertClaimableBalances(ctx context.Context, cbs []ClaimableBalance) error { + a := m.Called(ctx, cbs) + return a.Error(0) +} + func (m *MockQClaimableBalances) RemoveClaimableBalances(ctx context.Context, ids []string) (int64, error) { a := m.Called(ctx, ids) return a.Get(0).(int64), a.Error(1) diff --git a/services/horizon/internal/db2/history/offers.go b/services/horizon/internal/db2/history/offers.go index 98a08fef87..1dcfe25321 100644 --- a/services/horizon/internal/db2/history/offers.go +++ b/services/horizon/internal/db2/history/offers.go @@ -5,8 +5,8 @@ import ( "database/sql" sq "github.com/Masterminds/squirrel" - "github.com/jmoiron/sqlx" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" ) @@ -108,7 +108,7 @@ func (q *Q) StreamAllOffers(ctx context.Context, callback func(Offer) error) err } func (q *Q) streamAllOffersBatch(ctx context.Context, lastId int64, limit uint64, callback func(Offer) error) (int64, error) { - var rows *sqlx.Rows + var rows *db.Rows var err error rows, err = q.Query(ctx, selectOffers. diff --git a/services/horizon/internal/db2/history/transaction_test.go b/services/horizon/internal/db2/history/transaction_test.go index 4e85624701..65c6734644 100644 --- a/services/horizon/internal/db2/history/transaction_test.go +++ b/services/horizon/internal/db2/history/transaction_test.go @@ -892,12 +892,20 @@ func TestFetchFeeBumpTransaction(t *testing.T) { tt.Assert.Equal(byOuterhash, byInnerHash) } - var outerEffects, innerEffects []Effect - err = q.Effects().ForTransaction(tt.Ctx, fixture.OuterHash).Select(tt.Ctx, &outerEffects) + var innerEffects []Effect + outerEffects, err := q.EffectsForTransaction(tt.Ctx, fixture.OuterHash, db2.PageQuery{ + Cursor: "0-0", + Order: "asc", + Limit: 200, + }) tt.Assert.NoError(err) tt.Assert.Len(outerEffects, 1) - err = q.Effects().ForTransaction(tt.Ctx, fixture.InnerHash).Select(tt.Ctx, &innerEffects) + innerEffects, err = q.EffectsForTransaction(tt.Ctx, fixture.InnerHash, db2.PageQuery{ + Cursor: "0-0", + Order: "asc", + Limit: 200, + }) tt.Assert.NoError(err) tt.Assert.Equal(outerEffects, innerEffects) } diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 4c8e4dc2f5..5f11389a5f 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "strings" + "time" "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -68,7 +69,9 @@ const ( // StellarTestnet is a constant representing the Stellar test network StellarTestnet = "testnet" - defaultMaxHTTPRequestSize = uint(200 * 1024) + defaultMaxConcurrentRequests = uint(1000) + defaultMaxHTTPRequestSize = uint(200 * 1024) + clientQueryTimeoutNotSet = -1 ) var ( @@ -437,6 +440,30 @@ func Flags() (*Config, support.ConfigOptions) { Usage: "defines the timeout of connection after which 504 response will be sent or stream will be closed, if Horizon is behind a load balancer with idle connection timeout, this should be set to a few seconds less that idle timeout, does not apply to POST /transactions", UsedInCommands: ApiServerCommands, }, + &support.ConfigOption{ + Name: "client-query-timeout", + ConfigKey: &config.ClientQueryTimeout, + OptType: types.Int, + FlagDefault: clientQueryTimeoutNotSet, + CustomSetValue: func(co *support.ConfigOption) error { + if !support.IsExplicitlySet(co) { + *(co.ConfigKey.(*time.Duration)) = time.Duration(co.FlagDefault.(int)) + return nil + } + duration := viper.GetInt(co.Name) + if duration < 0 { + return fmt.Errorf("%s cannot be negative", co.Name) + } + *(co.ConfigKey.(*time.Duration)) = time.Duration(duration) * time.Second + return nil + }, + Usage: "defines the timeout for when horizon will cancel all postgres queries connected to an HTTP request. The timeout is measured in seconds since the start of the HTTP request. Note, this timeout does not apply to POST /transactions. " + + "The difference between client-query-timeout and connection-timeout is that connection-timeout applies a postgres statement timeout whereas client-query-timeout will send an additional request to postgres to cancel the ongoing query. " + + "Generally, client-query-timeout should be configured to be higher than connection-timeout to allow the postgres statement timeout to kill long running queries without having to send the additional cancel request to postgres. " + + "By default, client-query-timeout will be set to twice the connection-timeout. Setting client-query-timeout to 0 will disable the timeout which means that Horizon will never kill long running queries using the cancel request, however, " + + "long running queries can still be killed through the postgres statement timeout which is configured via the connection-timeout flag.", + UsedInCommands: ApiServerCommands, + }, &support.ConfigOption{ Name: "max-http-request-size", ConfigKey: &config.MaxHTTPRequestSize, @@ -445,6 +472,15 @@ func Flags() (*Config, support.ConfigOptions) { Usage: "sets the limit on the maximum allowed http request payload size, default is 200kb, to disable the limit check, set to 0, only do so if you acknowledge the implications of accepting unbounded http request payload sizes.", UsedInCommands: ApiServerCommands, }, + &support.ConfigOption{ + Name: "max-concurrent-requests", + ConfigKey: &config.MaxConcurrentRequests, + OptType: types.Uint, + FlagDefault: defaultMaxConcurrentRequests, + Usage: "sets the limit on the maximum number of concurrent http requests, default is 1000, to disable the limit set to 0. " + + "If Horizon receives a request which would exceed the limit of concurrent http requests, Horizon will respond with a 503 status code.", + UsedInCommands: ApiServerCommands, + }, &support.ConfigOption{ Name: "per-hour-rate-limit", ConfigKey: &config.RateQuota, @@ -983,5 +1019,10 @@ func ApplyFlags(config *Config, flags support.ConfigOptions, options ApplyOption " If Horizon is behind both, use --behind-cloudflare only") } + if config.ClientQueryTimeout == clientQueryTimeoutNotSet { + // the default value for cancel-db-query-timeout is twice the connection-timeout + config.ClientQueryTimeout = config.ConnectionTimeout * 2 + } + return nil } diff --git a/services/horizon/internal/flags_test.go b/services/horizon/internal/flags_test.go index ef2d5d3a02..3da39bc7a5 100644 --- a/services/horizon/internal/flags_test.go +++ b/services/horizon/internal/flags_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "testing" + "time" "github.com/spf13/cobra" @@ -207,22 +208,69 @@ func Test_createCaptiveCoreConfig(t *testing.T) { } } -func TestEnvironmentVariables(t *testing.T) { - environmentVars := map[string]string{ - "INGEST": "false", - "HISTORY_ARCHIVE_URLS": "http://localhost:1570", - "DATABASE_URL": "postgres://postgres@localhost/test_332cb65e6b00?sslmode=disable&timezone=UTC", - "STELLAR_CORE_URL": "http://localhost:11626", - "NETWORK_PASSPHRASE": "Standalone Network ; February 2017", - "APPLY_MIGRATIONS": "true", - "CHECKPOINT_FREQUENCY": "8", - "MAX_DB_CONNECTIONS": "50", - "ADMIN_PORT": "6060", - "PORT": "8001", - "CAPTIVE_CORE_BINARY_PATH": os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN"), - "CAPTIVE_CORE_CONFIG_PATH": "../docker/captive-core-classic-integration-tests.cfg", - "CAPTIVE_CORE_USE_DB": "true", +func TestClientQueryTimeoutFlag(t *testing.T) { + for _, testCase := range []struct { + name string + flag string + parsed time.Duration + err string + }{ + { + "negative value", + "-1", + 0, + "client-query-timeout cannot be negative", + }, + { + "default value", + "", + time.Second * 110, + "", + }, + { + "custom value", + "20", + time.Second * 20, + "", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + environmentVars := horizonEnvVars() + if testCase.flag != "" { + environmentVars["CLIENT_QUERY_TIMEOUT"] = testCase.flag + } + + envManager := test.NewEnvironmentManager() + defer func() { + envManager.Restore() + }() + if err := envManager.InitializeEnvironmentVariables(environmentVars); err != nil { + require.NoError(t, err) + } + + config, flags := Flags() + horizonCmd := &cobra.Command{ + Use: "horizon", + Short: "Client-facing api server for the Stellar network", + SilenceErrors: true, + SilenceUsage: true, + Long: "Client-facing API server for the Stellar network.", + } + if err := flags.Init(horizonCmd); err != nil { + require.NoError(t, err) + } + if err := ApplyFlags(config, flags, ApplyOptions{RequireCaptiveCoreFullConfig: true, AlwaysIngest: false}); err != nil { + require.EqualError(t, err, testCase.err) + } else { + require.Empty(t, testCase.err) + } + require.Equal(t, testCase.parsed, config.ClientQueryTimeout) + }) } +} + +func TestEnvironmentVariables(t *testing.T) { + environmentVars := horizonEnvVars() envManager := test.NewEnvironmentManager() defer func() { @@ -261,6 +309,24 @@ func TestEnvironmentVariables(t *testing.T) { assert.Equal(t, config.CaptiveCoreConfigUseDB, true) } +func horizonEnvVars() map[string]string { + return map[string]string{ + "INGEST": "false", + "HISTORY_ARCHIVE_URLS": "http://localhost:1570", + "DATABASE_URL": "postgres://postgres@localhost/test_332cb65e6b00?sslmode=disable&timezone=UTC", + "STELLAR_CORE_URL": "http://localhost:11626", + "NETWORK_PASSPHRASE": "Standalone Network ; February 2017", + "APPLY_MIGRATIONS": "true", + "CHECKPOINT_FREQUENCY": "8", + "MAX_DB_CONNECTIONS": "50", + "ADMIN_PORT": "6060", + "PORT": "8001", + "CAPTIVE_CORE_BINARY_PATH": os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN"), + "CAPTIVE_CORE_CONFIG_PATH": "../docker/captive-core-classic-integration-tests.cfg", + "CAPTIVE_CORE_USE_DB": "true", + } +} + func TestRemovedFlags(t *testing.T) { tests := []struct { name string diff --git a/services/horizon/internal/httpx/middleware.go b/services/horizon/internal/httpx/middleware.go index eaa49634a5..9240ed0833 100644 --- a/services/horizon/internal/httpx/middleware.go +++ b/services/horizon/internal/httpx/middleware.go @@ -197,7 +197,7 @@ func recoverMiddleware(h http.Handler) http.Handler { // NewHistoryMiddleware adds session to the request context and ensures Horizon // is not in a stale state, which is when the difference between latest core // ledger and latest history ledger is higher than the given threshold -func NewHistoryMiddleware(ledgerState *ledger.State, staleThreshold int32, session db.SessionInterface) func(http.Handler) http.Handler { +func NewHistoryMiddleware(ledgerState *ledger.State, staleThreshold int32, session db.SessionInterface, contextDBTimeout time.Duration) func(http.Handler) http.Handler { return func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -205,6 +205,7 @@ func NewHistoryMiddleware(ledgerState *ledger.State, staleThreshold int32, sessi if routePattern := supportHttp.GetChiRoutePattern(r); routePattern != "" { ctx = context.WithValue(ctx, &db.RouteContextKey, routePattern) } + ctx = setContextDBTimeout(contextDBTimeout, ctx) if staleThreshold > 0 { ls := ledgerState.CurrentStatus() isStale := (ls.CoreLatest - ls.HistoryLatest) > int32(staleThreshold) @@ -238,6 +239,7 @@ func NewHistoryMiddleware(ledgerState *ledger.State, staleThreshold int32, sessi // returning invalid data to the user) type StateMiddleware struct { HorizonSession db.SessionInterface + ClientQueryTimeout time.Duration NoStateVerification bool } @@ -276,6 +278,7 @@ func (m *StateMiddleware) WrapFunc(h http.HandlerFunc) http.HandlerFunc { if routePattern := supportHttp.GetChiRoutePattern(r); routePattern != "" { ctx = context.WithValue(ctx, &db.RouteContextKey, routePattern) } + ctx = setContextDBTimeout(m.ClientQueryTimeout, ctx) session := m.HorizonSession.Clone() q := &history.Q{session} sseRequest := render.Negotiate(r) == render.MimeEventStream @@ -344,6 +347,14 @@ func (m *StateMiddleware) WrapFunc(h http.HandlerFunc) http.HandlerFunc { } } +func setContextDBTimeout(timeout time.Duration, ctx context.Context) context.Context { + var deadline time.Time + if timeout > 0 { + deadline = time.Now().Add(timeout) + } + return context.WithValue(ctx, &db.DeadlineCtxKey, deadline) +} + // WrapFunc executes the middleware on a given HTTP handler function func (m *StateMiddleware) Wrap(h http.Handler) http.Handler { return m.WrapFunc(h.ServeHTTP) diff --git a/services/horizon/internal/httpx/router.go b/services/horizon/internal/httpx/router.go index 8fa57d0379..4ba978a96a 100644 --- a/services/horizon/internal/httpx/router.go +++ b/services/horizon/internal/httpx/router.go @@ -28,16 +28,18 @@ import ( ) type RouterConfig struct { - DBSession db.SessionInterface - PrimaryDBSession db.SessionInterface - TxSubmitter *txsub.System - RateQuota *throttled.RateQuota + DBSession db.SessionInterface + PrimaryDBSession db.SessionInterface + TxSubmitter *txsub.System + RateQuota *throttled.RateQuota + MaxConcurrentRequests uint BehindCloudflare bool BehindAWSLoadBalancer bool SSEUpdateFrequency time.Duration StaleThreshold uint ConnectionTimeout time.Duration + ClientQueryTimeout time.Duration MaxHTTPRequestSize uint NetworkPassphrase string MaxPathLength uint @@ -50,6 +52,7 @@ type RouterConfig struct { HealthCheck http.Handler EnableIngestionFiltering bool DisableTxSub bool + SkipTxMeta bool } type Router struct { @@ -89,6 +92,9 @@ func (r *Router) addMiddleware(config *RouterConfig, BehindAWSLoadBalancer: config.BehindAWSLoadBalancer, })) r.Use(loggerMiddleware(serverMetrics)) + if config.MaxConcurrentRequests > 0 { + r.Use(chimiddleware.Throttle(int(config.MaxConcurrentRequests))) + } r.Use(timeoutMiddleware(config.ConnectionTimeout)) if config.MaxHTTPRequestSize > 0 { r.Use(func(handler http.Handler) http.Handler { @@ -138,7 +144,8 @@ func (r *Router) addMiddleware(config *RouterConfig, func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRateLimiter, ledgerState *ledger.State) { stateMiddleware := StateMiddleware{ - HorizonSession: config.DBSession, + HorizonSession: config.DBSession, + ClientQueryTimeout: config.ClientQueryTimeout, } r.Method(http.MethodGet, "/health", config.HealthCheck) @@ -156,7 +163,7 @@ func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRate LedgerSourceFactory: historyLedgerSourceFactory{ledgerState: ledgerState, updateFrequency: config.SSEUpdateFrequency}, } - historyMiddleware := NewHistoryMiddleware(ledgerState, int32(config.StaleThreshold), config.DBSession) + historyMiddleware := NewHistoryMiddleware(ledgerState, int32(config.StaleThreshold), config.DBSession, config.ClientQueryTimeout) // State endpoints behind stateMiddleware r.Group(func(r chi.Router) { r.Route("/accounts", func(r chi.Router) { @@ -191,8 +198,9 @@ func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRate r.With(historyMiddleware).Method(http.MethodGet, "/operations", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: false, + SkipTxMeta: config.SkipTxMeta, }, streamHandler)) - r.With(historyMiddleware).Method(http.MethodGet, "/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState}, streamHandler)) + r.With(historyMiddleware).Method(http.MethodGet, "/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState, SkipTxMeta: config.SkipTxMeta}, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/effects", streamableHistoryPageHandler(ledgerState, actions.GetEffectsHandler{LedgerState: ledgerState}, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/trades", streamableHistoryPageHandler(ledgerState, actions.GetTradesHandler{LedgerState: ledgerState, CoreStateGetter: config.CoreGetter}, streamHandler)) }) @@ -241,29 +249,32 @@ func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRate r.With(historyMiddleware).Method(http.MethodGet, "/accounts/{account_id:\\w+}/operations", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: false, + SkipTxMeta: config.SkipTxMeta, }, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/accounts/{account_id:\\w+}/payments", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: true, }, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/accounts/{account_id:\\w+}/trades", streamableHistoryPageHandler(ledgerState, actions.GetTradesHandler{LedgerState: ledgerState, CoreStateGetter: config.CoreGetter}, streamHandler)) - r.With(historyMiddleware).Method(http.MethodGet, "/accounts/{account_id:\\w+}/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState}, streamHandler)) + r.With(historyMiddleware).Method(http.MethodGet, "/accounts/{account_id:\\w+}/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState, SkipTxMeta: config.SkipTxMeta}, streamHandler)) }) // ledger actions r.Route("/ledgers", func(r chi.Router) { r.With(historyMiddleware).Method(http.MethodGet, "/", streamableHistoryPageHandler(ledgerState, actions.GetLedgersHandler{LedgerState: ledgerState}, streamHandler)) r.Route("/{ledger_id}", func(r chi.Router) { r.With(historyMiddleware).Method(http.MethodGet, "/", ObjectActionHandler{actions.GetLedgerByIDHandler{LedgerState: ledgerState}}) - r.With(historyMiddleware).Method(http.MethodGet, "/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState}, streamHandler)) + r.With(historyMiddleware).Method(http.MethodGet, "/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState, SkipTxMeta: config.SkipTxMeta}, streamHandler)) r.Group(func(r chi.Router) { r.With(historyMiddleware).Method(http.MethodGet, "/effects", streamableHistoryPageHandler(ledgerState, actions.GetEffectsHandler{LedgerState: ledgerState}, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/operations", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: false, + SkipTxMeta: config.SkipTxMeta, }, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/payments", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: true, + SkipTxMeta: config.SkipTxMeta, }, streamHandler)) }) }) @@ -275,18 +286,19 @@ func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRate LedgerState: ledgerState, OnlyPayments: false, }, streamHandler)) - r.With(historyMiddleware).Method(http.MethodGet, "/claimable_balances/{claimable_balance_id:\\w+}/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState}, streamHandler)) + r.With(historyMiddleware).Method(http.MethodGet, "/claimable_balances/{claimable_balance_id:\\w+}/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState, SkipTxMeta: config.SkipTxMeta}, streamHandler)) }) // transaction history actions r.Route("/transactions", func(r chi.Router) { - r.With(historyMiddleware).Method(http.MethodGet, "/", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState}, streamHandler)) + r.With(historyMiddleware).Method(http.MethodGet, "/", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState, SkipTxMeta: config.SkipTxMeta}, streamHandler)) r.Route("/{tx_id}", func(r chi.Router) { r.With(historyMiddleware).Method(http.MethodGet, "/", ObjectActionHandler{actions.GetTransactionByHashHandler{}}) r.With(historyMiddleware).Method(http.MethodGet, "/effects", streamableHistoryPageHandler(ledgerState, actions.GetEffectsHandler{LedgerState: ledgerState}, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/operations", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: false, + SkipTxMeta: config.SkipTxMeta, }, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/payments", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, @@ -300,8 +312,9 @@ func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRate r.With(historyMiddleware).Method(http.MethodGet, "/", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: false, + SkipTxMeta: config.SkipTxMeta, }, streamHandler)) - r.With(historyMiddleware).Method(http.MethodGet, "/{id}", ObjectActionHandler{actions.GetOperationByIDHandler{LedgerState: ledgerState}}) + r.With(historyMiddleware).Method(http.MethodGet, "/{id}", ObjectActionHandler{actions.GetOperationByIDHandler{LedgerState: ledgerState, SkipTxMeta: config.SkipTxMeta}}) r.With(historyMiddleware).Method(http.MethodGet, "/{op_id}/effects", streamableHistoryPageHandler(ledgerState, actions.GetEffectsHandler{LedgerState: ledgerState}, streamHandler)) }) @@ -329,6 +342,7 @@ func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRate NetworkPassphrase: config.NetworkPassphrase, DisableTxSub: config.DisableTxSub, CoreStateGetter: config.CoreGetter, + SkipTxMeta: config.SkipTxMeta, }}) // Network state related endpoints diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go index de729f9605..fce002881c 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go @@ -2,7 +2,6 @@ package processors import ( "context" - "fmt" "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" @@ -60,7 +59,8 @@ func (p *ClaimableBalancesChangeProcessor) ProcessChange(ctx context.Context, ch func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { defer p.reset() var ( - cbIDsToDelete []string + cbIDsToDelete []string + updatedBalances []history.ClaimableBalance ) changes := p.cache.GetChanges() for _, change := range changes { @@ -97,8 +97,13 @@ func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { } cbIDsToDelete = append(cbIDsToDelete, id) default: - // claimable balance can only be created or removed - return fmt.Errorf("invalid change entry for a claimable balance was detected") + // this case should only occur if the sponsor has changed in the claimable balance + // the other fields of a claimable balance are immutable + postCB, err := p.ledgerEntryToRow(change.Post) + if err != nil { + return err + } + updatedBalances = append(updatedBalances, postCB) } } @@ -112,6 +117,12 @@ func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { return errors.Wrap(err, "error executing ClaimableBalanceBatchInsertBuilder") } + if len(updatedBalances) > 0 { + if err = p.qClaimableBalances.UpsertClaimableBalances(ctx, updatedBalances); err != nil { + return errors.Wrap(err, "error updating claimable balances") + } + } + if len(cbIDsToDelete) > 0 { count, err := p.qClaimableBalances.RemoveClaimableBalances(ctx, cbIDsToDelete) if err != nil { diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go index 524de095f7..a10cc9db7d 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go @@ -8,10 +8,11 @@ import ( "github.com/guregu/null" + "github.com/stretchr/testify/suite" + "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/xdr" - "github.com/stretchr/testify/suite" ) func TestClaimableBalancesChangeProcessorTestSuiteState(t *testing.T) { @@ -249,3 +250,71 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TestRemoveClaimableBal []string{id}, ).Return(int64(1), nil).Once() } + +func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TestUpdateClaimableBalanceAddSponsor() { + balanceID := xdr.ClaimableBalanceId{ + Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0, + V0: &xdr.Hash{1, 2, 3}, + } + cBalance := xdr.ClaimableBalanceEntry{ + BalanceId: balanceID, + Claimants: []xdr.Claimant{}, + Asset: xdr.MustNewCreditAsset("USD", "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), + Amount: 10, + } + lastModifiedLedgerSeq := xdr.Uint32(123) + + pre := xdr.LedgerEntry{ + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeClaimableBalance, + ClaimableBalance: &cBalance, + }, + LastModifiedLedgerSeq: lastModifiedLedgerSeq - 1, + Ext: xdr.LedgerEntryExt{ + V: 1, + V1: &xdr.LedgerEntryExtensionV1{ + SponsoringId: nil, + }, + }, + } + + // add sponsor + updated := xdr.LedgerEntry{ + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeClaimableBalance, + ClaimableBalance: &cBalance, + }, + LastModifiedLedgerSeq: lastModifiedLedgerSeq, + Ext: xdr.LedgerEntryExt{ + V: 1, + V1: &xdr.LedgerEntryExtensionV1{ + SponsoringId: xdr.MustAddressPtr("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), + }, + }, + } + s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() + + err := s.processor.ProcessChange(s.ctx, ingest.Change{ + Type: xdr.LedgerEntryTypeClaimableBalance, + Pre: &pre, + Post: &updated, + }) + s.Assert().NoError(err) + + id, err := xdr.MarshalHex(balanceID) + s.Assert().NoError(err) + s.mockQ.On( + "UpsertClaimableBalances", + s.ctx, + []history.ClaimableBalance{ + { + BalanceID: id, + Claimants: []history.Claimant{}, + Asset: cBalance.Asset, + Amount: cBalance.Amount, + LastModifiedLedger: uint32(lastModifiedLedgerSeq), + Sponsor: null.StringFrom("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), + }, + }, + ).Return(nil).Once() +} diff --git a/services/horizon/internal/init.go b/services/horizon/internal/init.go index 0c0fe2c2cd..2311833508 100644 --- a/services/horizon/internal/init.go +++ b/services/horizon/internal/init.go @@ -45,38 +45,29 @@ func mustInitHorizonDB(app *App) { log.Fatalf("max open connections to horizon db must be greater than %d", ingest.MaxDBConnections) } } + serverSidePGTimeoutConfigs := []db.ClientConfig{ + db.StatementTimeout(app.config.ConnectionTimeout), + db.IdleTransactionTimeout(app.config.ConnectionTimeout), + } if app.config.RoDatabaseURL == "" { - var clientConfigs []db.ClientConfig - if !app.config.Ingest { - // if we are not ingesting then we don't expect to have long db queries / transactions - clientConfigs = append( - clientConfigs, - db.StatementTimeout(app.config.ConnectionTimeout), - db.IdleTransactionTimeout(app.config.ConnectionTimeout), - ) - } app.historyQ = &history.Q{mustNewDBSession( db.HistorySubservice, app.config.DatabaseURL, maxIdle, maxOpen, app.prometheusRegistry, - clientConfigs..., + serverSidePGTimeoutConfigs..., )} } else { // If RO set, use it for all DB queries - roClientConfigs := []db.ClientConfig{ - db.StatementTimeout(app.config.ConnectionTimeout), - db.IdleTransactionTimeout(app.config.ConnectionTimeout), - } app.historyQ = &history.Q{mustNewDBSession( db.HistorySubservice, app.config.RoDatabaseURL, maxIdle, maxOpen, app.prometheusRegistry, - roClientConfigs..., + serverSidePGTimeoutConfigs..., )} app.primaryHistoryQ = &history.Q{mustNewDBSession( @@ -85,6 +76,7 @@ func mustInitHorizonDB(app *App) { maxIdle, maxOpen, app.prometheusRegistry, + serverSidePGTimeoutConfigs..., )} } } diff --git a/services/horizon/internal/integration/transaction_test.go b/services/horizon/internal/integration/transaction_test.go index 85fbe78522..a0db5816dd 100644 --- a/services/horizon/internal/integration/transaction_test.go +++ b/services/horizon/internal/integration/transaction_test.go @@ -63,13 +63,7 @@ func TestP19MetaDisabledTransaction(t *testing.T) { clientTx := itest.MustSubmitOperations(&masterAccount, itest.Master(), op) - var txMetaResult xdr.TransactionMeta - err = xdr.SafeUnmarshalBase64(clientTx.ResultMetaXdr, &txMetaResult) - require.NoError(t, err) - - assert.Equal(t, len(txMetaResult.MustV2().Operations), 0) - assert.Equal(t, len(txMetaResult.MustV2().TxChangesAfter), 0) - assert.Equal(t, len(txMetaResult.MustV2().TxChangesBefore), 0) + assert.Empty(t, clientTx.ResultMetaXdr) } func TestP20MetaTransaction(t *testing.T) { @@ -123,12 +117,5 @@ func TestP20MetaDisabledTransaction(t *testing.T) { preFlightOp, minFee := itest.PreflightHostFunctions(&sourceAccount, *installContractOp) clientTx := itest.MustSubmitOperationsWithFee(&sourceAccount, itest.Master(), minFee+txnbuild.MinBaseFee, &preFlightOp) - var txMetaResult xdr.TransactionMeta - err = xdr.SafeUnmarshalBase64(clientTx.ResultMetaXdr, &txMetaResult) - require.NoError(t, err) - - assert.Equal(t, len(txMetaResult.MustV3().Operations), 0) - assert.Nil(t, txMetaResult.MustV3().SorobanMeta) - assert.Equal(t, len(txMetaResult.MustV3().TxChangesAfter), 0) - assert.Equal(t, len(txMetaResult.MustV3().TxChangesBefore), 0) + assert.Empty(t, clientTx.ResultMetaXdr) } diff --git a/services/horizon/internal/middleware_test.go b/services/horizon/internal/middleware_test.go index 40a74ea69e..269269bf8e 100644 --- a/services/horizon/internal/middleware_test.go +++ b/services/horizon/internal/middleware_test.go @@ -402,7 +402,7 @@ func TestCheckHistoryStaleMiddleware(t *testing.T) { } ledgerState := &ledger.State{} ledgerState.SetStatus(state) - historyMiddleware := httpx.NewHistoryMiddleware(ledgerState, testCase.staleThreshold, tt.HorizonSession()) + historyMiddleware := httpx.NewHistoryMiddleware(ledgerState, testCase.staleThreshold, tt.HorizonSession(), 0) handler := chi.NewRouter() handler.With(historyMiddleware).MethodFunc("GET", "/", endpoint) w := httptest.NewRecorder() diff --git a/services/horizon/internal/resourceadapter/operations.go b/services/horizon/internal/resourceadapter/operations.go index 2f995fb395..fc301aec64 100644 --- a/services/horizon/internal/resourceadapter/operations.go +++ b/services/horizon/internal/resourceadapter/operations.go @@ -20,10 +20,11 @@ func NewOperation( transactionHash string, transactionRow *history.Transaction, ledger history.Ledger, + skipTxMeta bool, ) (result hal.Pageable, err error) { base := operations.Base{} - err = PopulateBaseOperation(ctx, &base, operationRow, transactionHash, transactionRow, ledger) + err = PopulateBaseOperation(ctx, &base, operationRow, transactionHash, transactionRow, ledger, skipTxMeta) if err != nil { return } @@ -166,7 +167,7 @@ func NewOperation( } // Populate fills out this resource using `row` as the source. -func PopulateBaseOperation(ctx context.Context, dest *operations.Base, operationRow history.Operation, transactionHash string, transactionRow *history.Transaction, ledger history.Ledger) error { +func PopulateBaseOperation(ctx context.Context, dest *operations.Base, operationRow history.Operation, transactionHash string, transactionRow *history.Transaction, ledger history.Ledger, skipTxMeta bool) error { dest.ID = fmt.Sprintf("%d", operationRow.ID) dest.PT = operationRow.PagingToken() dest.TransactionSuccessful = operationRow.TransactionSuccessful @@ -190,7 +191,7 @@ func PopulateBaseOperation(ctx context.Context, dest *operations.Base, operation if transactionRow != nil { dest.Transaction = new(horizon.Transaction) - return PopulateTransaction(ctx, transactionHash, dest.Transaction, *transactionRow) + return PopulateTransaction(ctx, transactionHash, dest.Transaction, *transactionRow, skipTxMeta) } return nil } diff --git a/services/horizon/internal/resourceadapter/operations_test.go b/services/horizon/internal/resourceadapter/operations_test.go index 39660a3678..f6acfa664b 100644 --- a/services/horizon/internal/resourceadapter/operations_test.go +++ b/services/horizon/internal/resourceadapter/operations_test.go @@ -19,7 +19,7 @@ func TestNewOperationAllTypesCovered(t *testing.T) { row := history.Operation{ Type: xdr.OperationType(typ), } - op, err := NewOperation(context.Background(), row, "foo", tx, history.Ledger{}) + op, err := NewOperation(context.Background(), row, "foo", tx, history.Ledger{}, false) assert.NoError(t, err, s) // if we got a base type, the operation is not covered if _, ok := op.(operations.Base); ok { @@ -31,7 +31,7 @@ func TestNewOperationAllTypesCovered(t *testing.T) { row := history.Operation{ Type: xdr.OperationType(200000), } - op, err := NewOperation(context.Background(), row, "foo", tx, history.Ledger{}) + op, err := NewOperation(context.Background(), row, "foo", tx, history.Ledger{}, false) assert.NoError(t, err) assert.IsType(t, op, operations.Base{}) @@ -52,7 +52,7 @@ func TestPopulateOperation_Successful(t *testing.T) { assert.NoError( t, - PopulateBaseOperation(ctx, &dest, row, "", nil, ledger), + PopulateBaseOperation(ctx, &dest, row, "", nil, ledger, false), ) assert.True(t, dest.TransactionSuccessful) assert.Nil(t, dest.Transaction) @@ -62,7 +62,7 @@ func TestPopulateOperation_Successful(t *testing.T) { assert.NoError( t, - PopulateBaseOperation(ctx, &dest, row, "", nil, ledger), + PopulateBaseOperation(ctx, &dest, row, "", nil, ledger, false), ) assert.False(t, dest.TransactionSuccessful) assert.Nil(t, dest.Transaction) @@ -92,12 +92,13 @@ func TestPopulateOperation_WithTransaction(t *testing.T) { assert.NoError( t, - PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.TransactionHash, &transactionRow, ledger), + PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.TransactionHash, &transactionRow, ledger, true), ) assert.True(t, dest.TransactionSuccessful) assert.True(t, dest.Transaction.Successful) assert.Equal(t, int64(100), dest.Transaction.FeeCharged) assert.Equal(t, int64(10000), dest.Transaction.MaxFee) + assert.Empty(t, dest.Transaction.ResultMetaXdr) } func TestPopulateOperation_AllowTrust(t *testing.T) { @@ -308,7 +309,7 @@ func getJSONResponse(typ xdr.OperationType, details string) (rsp map[string]inte Type: typ, DetailsString: null.StringFrom(details), } - resource, err := NewOperation(ctx, operationsRow, "", &transactionRow, history.Ledger{}) + resource, err := NewOperation(ctx, operationsRow, "", &transactionRow, history.Ledger{}, false) if err != nil { return } @@ -343,19 +344,19 @@ func TestFeeBumpOperation(t *testing.T) { assert.NoError( t, - PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.TransactionHash, nil, history.Ledger{}), + PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.TransactionHash, nil, history.Ledger{}, false), ) assert.Equal(t, transactionRow.TransactionHash, dest.TransactionHash) assert.NoError( t, - PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.InnerTransactionHash.String, nil, history.Ledger{}), + PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.InnerTransactionHash.String, nil, history.Ledger{}, false), ) assert.Equal(t, transactionRow.InnerTransactionHash.String, dest.TransactionHash) assert.NoError( t, - PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.TransactionHash, &transactionRow, history.Ledger{}), + PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.TransactionHash, &transactionRow, history.Ledger{}, false), ) assert.Equal(t, transactionRow.TransactionHash, dest.TransactionHash) @@ -374,7 +375,7 @@ func TestFeeBumpOperation(t *testing.T) { assert.NoError( t, - PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.InnerTransactionHash.String, &transactionRow, history.Ledger{}), + PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.InnerTransactionHash.String, &transactionRow, history.Ledger{}, false), ) assert.Equal(t, transactionRow.InnerTransactionHash.String, dest.TransactionHash) assert.Equal(t, transactionRow.InnerTransactionHash.String, dest.Transaction.Hash) diff --git a/services/horizon/internal/resourceadapter/transaction.go b/services/horizon/internal/resourceadapter/transaction.go index 547f8f4b40..f458830c2d 100644 --- a/services/horizon/internal/resourceadapter/transaction.go +++ b/services/horizon/internal/resourceadapter/transaction.go @@ -23,6 +23,7 @@ func PopulateTransaction( transactionHash string, dest *protocol.Transaction, row history.Transaction, + skipTxMeta bool, ) error { dest.ID = transactionHash dest.PT = row.PagingToken() @@ -43,7 +44,11 @@ func PopulateTransaction( dest.OperationCount = row.OperationCount dest.EnvelopeXdr = row.TxEnvelope dest.ResultXdr = row.TxResult - dest.ResultMetaXdr = row.TxMeta + if skipTxMeta { + dest.ResultMetaXdr = "" + } else { + dest.ResultMetaXdr = row.TxMeta + } dest.FeeMetaXdr = row.TxFeeMeta dest.MemoType = row.MemoType dest.Memo = row.Memo.String diff --git a/services/horizon/internal/resourceadapter/transaction_test.go b/services/horizon/internal/resourceadapter/transaction_test.go index 29c8040ce6..694fc885fb 100644 --- a/services/horizon/internal/resourceadapter/transaction_test.go +++ b/services/horizon/internal/resourceadapter/transaction_test.go @@ -34,18 +34,40 @@ func TestPopulateTransaction_Successful(t *testing.T) { }, } - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) assert.True(t, dest.Successful) dest = Transaction{} row = history.Transaction{ TransactionWithoutLedger: history.TransactionWithoutLedger{ Successful: false, + TxMeta: "xyz", }, } - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) assert.False(t, dest.Successful) + assert.NotEmpty(t, dest.ResultMetaXdr) +} + +func TestPopulateTransactionWhenSkipMeta(t *testing.T) { + ctx, _ := test.ContextWithLogBuffer() + + var ( + dest Transaction + row history.Transaction + ) + + dest = Transaction{} + row = history.Transaction{ + TransactionWithoutLedger: history.TransactionWithoutLedger{ + Successful: true, + }, + } + + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, true)) + assert.True(t, dest.Successful) + assert.Empty(t, dest.ResultMetaXdr) } func TestPopulateTransaction_HashMemo(t *testing.T) { @@ -55,12 +77,14 @@ func TestPopulateTransaction_HashMemo(t *testing.T) { TransactionWithoutLedger: history.TransactionWithoutLedger{ MemoType: "hash", Memo: null.StringFrom("abcdef"), + TxMeta: "xyz", }, } - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) assert.Equal(t, "hash", dest.MemoType) assert.Equal(t, "abcdef", dest.Memo) assert.Equal(t, "", dest.MemoBytes) + assert.NotEmpty(t, dest.ResultMetaXdr) } func TestPopulateTransaction_TextMemo(t *testing.T) { @@ -122,15 +146,17 @@ func TestPopulateTransaction_TextMemo(t *testing.T) { MemoType: "text", TxEnvelope: envelopeXDR, Memo: null.StringFrom("sample"), + TxMeta: "xyz", }, } var dest Transaction - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) assert.Equal(t, "text", dest.MemoType) assert.Equal(t, "sample", dest.Memo) assert.Equal(t, base64.StdEncoding.EncodeToString(rawMemo), dest.MemoBytes) + assert.NotEmpty(t, dest.ResultMetaXdr) } } @@ -148,12 +174,14 @@ func TestPopulateTransaction_Fee(t *testing.T) { TransactionWithoutLedger: history.TransactionWithoutLedger{ MaxFee: 10000, FeeCharged: 100, + TxMeta: "xyz", }, } - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) assert.Equal(t, int64(100), dest.FeeCharged) assert.Equal(t, int64(10000), dest.MaxFee) + assert.NotEmpty(t, dest.ResultMetaXdr) } // TestPopulateTransaction_Preconditions tests transaction object population. @@ -188,10 +216,12 @@ func TestPopulateTransaction_Preconditions(t *testing.T) { MinAccountSequenceAge: null.StringFrom(fmt.Sprint(minSequenceAge)), MinAccountSequenceLedgerGap: null.IntFrom(int64(minSequenceLedgerGap)), ExtraSigners: pq.StringArray{"D34DB33F", "8BADF00D"}, + TxMeta: "xyz", }, } - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) + assert.NotEmpty(t, dest.ResultMetaXdr) p := dest.Preconditions assert.Equal(t, validAfter.Format(time.RFC3339), dest.ValidAfter) assert.Equal(t, validBefore.Format(time.RFC3339), dest.ValidBefore) @@ -282,11 +312,13 @@ func TestPopulateTransaction_PreconditionsV2(t *testing.T) { Upper: null.IntFrom(int64(envelopeTimebounds.MaxTime)), }, TxEnvelope: envelopeXDR, + TxMeta: "xyz", }, } var dest Transaction - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) + assert.NotEmpty(t, dest.ResultMetaXdr) gotTimebounds := dest.Preconditions.TimeBounds assert.Equal(t, "5", gotTimebounds.MinTime) @@ -307,7 +339,7 @@ func TestPopulateTransaction_PreconditionsV2_Omissions(t *testing.T) { generic := map[string]interface{}{} row := history.Transaction{TransactionWithoutLedger: tx} - tt.NoError(PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + tt.NoError(PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) bytes, err := dest.MarshalJSON() tt.NoError(err) @@ -325,6 +357,7 @@ func TestPopulateTransaction_PreconditionsV2_Omissions(t *testing.T) { MinAccountSequence: null.IntFromPtr(nil), MinAccountSequenceAge: null.StringFrom("0"), ExtraSigners: pq.StringArray{}, + TxMeta: "xyz", }, { AccountSequence: 1, MinAccountSequenceLedgerGap: null.IntFrom(0), @@ -333,6 +366,7 @@ func TestPopulateTransaction_PreconditionsV2_Omissions(t *testing.T) { MinAccountSequence: null.IntFromPtr(nil), MinAccountSequenceAge: null.StringFromPtr(nil), ExtraSigners: nil, + TxMeta: "xyz", }, } { dest, js := jsonifyTx(tx) @@ -354,6 +388,7 @@ func TestPopulateTransaction_PreconditionsV2_Omissions(t *testing.T) { // exist entirely. tx.MinAccountSequenceLedgerGap = null.IntFromPtr(nil) dest, js = jsonifyTx(tx) + assert.NotEmpty(t, dest.ResultMetaXdr) tt.NotContains(js, "preconditions") } } @@ -375,10 +410,12 @@ func TestFeeBumpTransaction(t *testing.T) { InnerTransactionHash: null.StringFrom("2374e99349b9ef7dba9a5db3339b78fda8f34777b1af33ba468ad5c0df946d4d"), Signatures: []string{"a", "b", "c"}, InnerSignatures: []string{"d", "e", "f"}, + TxMeta: "xyz", }, } - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) + assert.NotEmpty(t, dest.ResultMetaXdr) assert.Equal(t, row.TransactionHash, dest.Hash) assert.Equal(t, row.TransactionHash, dest.ID) assert.Equal(t, row.FeeAccount.String, dest.FeeAccount) @@ -397,7 +434,8 @@ func TestFeeBumpTransaction(t *testing.T) { assert.Equal(t, []string{"a", "b", "c"}, dest.FeeBumpTransaction.Signatures) assert.Equal(t, "/transactions/"+row.TransactionHash, dest.Links.Transaction.Href) - assert.NoError(t, PopulateTransaction(ctx, row.InnerTransactionHash.String, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.InnerTransactionHash.String, &dest, row, false)) + assert.NotEmpty(t, dest.ResultMetaXdr) assert.Equal(t, row.InnerTransactionHash.String, dest.Hash) assert.Equal(t, row.InnerTransactionHash.String, dest.ID) assert.Equal(t, row.FeeAccount.String, dest.FeeAccount) diff --git a/services/horizon/internal/scripts/check_release_hash/Dockerfile b/services/horizon/internal/scripts/check_release_hash/Dockerfile index 6a054aab34..08f818dad3 100644 --- a/services/horizon/internal/scripts/check_release_hash/Dockerfile +++ b/services/horizon/internal/scripts/check_release_hash/Dockerfile @@ -1,5 +1,5 @@ # Change to Go version used in CI or rebuild with --build-arg. -ARG GO_IMAGE=golang:1.20-bullseye +ARG GO_IMAGE=golang:1.22-bullseye FROM $GO_IMAGE WORKDIR /go/src/github.com/stellar/go diff --git a/staticcheck.sh b/staticcheck.sh index 539641a4b3..5e07d0d026 100755 --- a/staticcheck.sh +++ b/staticcheck.sh @@ -1,7 +1,7 @@ #! /bin/bash set -e -version='2023.1.1' +version='2023.1.7' staticcheck='go run honnef.co/go/tools/cmd/staticcheck@'"$version" diff --git a/support/config/config_option.go b/support/config/config_option.go index 97df9ef956..1fab1910d7 100644 --- a/support/config/config_option.go +++ b/support/config/config_option.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" + "github.com/stellar/go/support/errors" "github.com/stellar/go/support/strutils" ) @@ -248,13 +249,12 @@ func parseEnvVars(entries []string) map[string]bool { return set } -var envVars = parseEnvVars(os.Environ()) - // IsExplicitlySet returns true if and only if the given config option was set explicitly either // via a command line argument or via an environment variable func IsExplicitlySet(co *ConfigOption) bool { // co.flag.Changed is only set to true when the configuration is set via command line parameter. // In the case where a variable is configured via environment variable we need to check envVars. + envVars := parseEnvVars(os.Environ()) return co.flag.Changed || envVars[co.EnvVar] } diff --git a/support/config/config_option_test.go b/support/config/config_option_test.go index 30be01424b..418a50267e 100644 --- a/support/config/config_option_test.go +++ b/support/config/config_option_test.go @@ -88,13 +88,8 @@ func TestConfigOption_optionalFlags_env_set_empty(t *testing.T) { } configOpts.Init(cmd) - prev := envVars - envVars = map[string]bool{ - "STRING": true, - } - defer func() { - envVars = prev - }() + defer os.Setenv("STRING", os.Getenv("STRING")) + os.Setenv("STRING", "") cmd.Execute() assert.Equal(t, "", *optString) @@ -118,15 +113,6 @@ func TestConfigOption_optionalFlags_env_set(t *testing.T) { } configOpts.Init(cmd) - prev := envVars - envVars = map[string]bool{ - "STRING": true, - "UINT": true, - } - defer func() { - envVars = prev - }() - defer os.Setenv("STRING", os.Getenv("STRING")) defer os.Setenv("UINT", os.Getenv("UINT")) os.Setenv("STRING", "str") diff --git a/support/db/main.go b/support/db/main.go index dca23526ee..4b0b4c8b84 100644 --- a/support/db/main.go +++ b/support/db/main.go @@ -21,6 +21,7 @@ import ( "github.com/Masterminds/squirrel" "github.com/jmoiron/sqlx" + "github.com/stellar/go/support/errors" // Enable postgres @@ -119,6 +120,7 @@ type Session struct { DB *sqlx.DB tx *sqlx.Tx + txCancel context.CancelFunc txOptions *sql.TxOptions errorHandlers []ErrorHandlerFunc } @@ -140,8 +142,8 @@ type SessionInterface interface { GetRaw(ctx context.Context, dest interface{}, query string, args ...interface{}) error Select(ctx context.Context, dest interface{}, query squirrel.Sqlizer) error SelectRaw(ctx context.Context, dest interface{}, query string, args ...interface{}) error - Query(ctx context.Context, query squirrel.Sqlizer) (*sqlx.Rows, error) - QueryRaw(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) + Query(ctx context.Context, query squirrel.Sqlizer) (*Rows, error) + QueryRaw(ctx context.Context, query string, args ...interface{}) (*Rows, error) GetTable(name string) *Table Exec(ctx context.Context, query squirrel.Sqlizer) (sql.Result, error) ExecRaw(ctx context.Context, query string, args ...interface{}) (sql.Result, error) diff --git a/support/db/mock_session.go b/support/db/mock_session.go index ce932cdbb3..e6cb58c987 100644 --- a/support/db/mock_session.go +++ b/support/db/mock_session.go @@ -72,14 +72,14 @@ func (m *MockSession) GetRaw(ctx context.Context, dest interface{}, query string return argss.Error(0) } -func (m *MockSession) Query(ctx context.Context, query squirrel.Sqlizer) (*sqlx.Rows, error) { +func (m *MockSession) Query(ctx context.Context, query squirrel.Sqlizer) (*Rows, error) { args := m.Called(ctx, query) - return args.Get(0).(*sqlx.Rows), args.Error(1) + return args.Get(0).(*Rows), args.Error(1) } -func (m *MockSession) QueryRaw(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) { +func (m *MockSession) QueryRaw(ctx context.Context, query string, args ...interface{}) (*Rows, error) { argss := m.Called(ctx, query, args) - return argss.Get(0).(*sqlx.Rows), argss.Error(1) + return argss.Get(0).(*Rows), argss.Error(1) } func (m *MockSession) Select(ctx context.Context, dest interface{}, query squirrel.Sqlizer) error { diff --git a/support/db/session.go b/support/db/session.go index 472fc40a37..6b5c2b18c0 100644 --- a/support/db/session.go +++ b/support/db/session.go @@ -12,28 +12,71 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/jmoiron/sqlx" "github.com/lib/pq" + "github.com/stellar/go/support/db/sqlutils" "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" ) +var DeadlineCtxKey = CtxKey("deadline") + +func noop() {} + +// context() checks if there is a override on the context timeout which is configured using DeadlineCtxKey. +// If the override exists, we return a new context with the desired deadline. Otherwise, we return the +// original context. +// Note that the override will not be applied if requestCtx has already been terminated. +// The timeout can be disabled by setting the DeadlineCtxKey value to a zero time.Time value, +// in that case the query will never be canceled. +func (s *Session) context(requestCtx context.Context) (context.Context, context.CancelFunc, error) { + var ctx context.Context + var cancel context.CancelFunc + + // if there is no DeadlineCtxKey value in the context we default to using the request context. + // this case is expected during ingestion where we don't want any queries to be canceled unless + // horizon is shutting down. + deadline, ok := requestCtx.Value(&DeadlineCtxKey).(time.Time) + if !ok { + return requestCtx, noop, nil + } + + // if requestCtx is already terminated don't proceed with the db statement + if requestCtx.Err() != nil { + return requestCtx, nil, requestCtx.Err() + } + + if deadline.IsZero() { + ctx, cancel = context.Background(), noop + } else { + ctx, cancel = context.WithDeadline(context.Background(), deadline) + } + return ctx, cancel, nil +} + // Begin binds this session to a new transaction. func (s *Session) Begin(ctx context.Context) error { if s.tx != nil { return errors.New("already in transaction") } + ctx, cancel, err := s.context(ctx) + if err != nil { + return err + } tx, err := s.DB.BeginTxx(ctx, nil) if err != nil { if knownErr := s.handleError(err, ctx); knownErr != nil { + cancel() return knownErr } + cancel() return errors.Wrap(err, "beginx failed") } log.Debug("sql: begin") s.tx = tx s.txOptions = nil + s.txCancel = cancel return nil } @@ -43,19 +86,26 @@ func (s *Session) BeginTx(ctx context.Context, opts *sql.TxOptions) error { if s.tx != nil { return errors.New("already in transaction") } + ctx, cancel, err := s.context(ctx) + if err != nil { + return err + } tx, err := s.DB.BeginTxx(ctx, opts) if err != nil { if knownErr := s.handleError(err, ctx); knownErr != nil { + cancel() return knownErr } + cancel() return errors.Wrap(err, "beginTx failed") } log.Debug("sql: begin") s.tx = tx s.txOptions = opts + s.txCancel = cancel return nil } @@ -93,6 +143,8 @@ func (s *Session) Commit() error { log.Debug("sql: commit") s.tx = nil s.txOptions = nil + s.txCancel() + s.txCancel = nil if knownErr := s.handleError(err, context.Background()); knownErr != nil { return knownErr @@ -135,7 +187,13 @@ func (s *Session) Get(ctx context.Context, dest interface{}, query sq.Sqlizer) e // GetRaw runs `query` with `args`, setting the first result found on // `dest`, if any. func (s *Session) GetRaw(ctx context.Context, dest interface{}, query string, args ...interface{}) error { - query, err := s.ReplacePlaceholders(query) + ctx, cancel, err := s.context(ctx) + if err != nil { + return err + } + defer cancel() + + query, err = s.ReplacePlaceholders(query) if err != nil { return errors.Wrap(err, "replace placeholders failed") } @@ -204,7 +262,13 @@ func (s *Session) ExecAll(ctx context.Context, script string) error { // ExecRaw runs `query` with `args` func (s *Session) ExecRaw(ctx context.Context, query string, args ...interface{}) (sql.Result, error) { - query, err := s.ReplacePlaceholders(query) + ctx, cancel, err := s.context(ctx) + if err != nil { + return nil, err + } + defer cancel() + + query, err = s.ReplacePlaceholders(query) if err != nil { return nil, errors.Wrap(err, "replace placeholders failed") } @@ -294,7 +358,7 @@ func (s *Session) handleError(dbErr error, ctx context.Context) error { } // Query runs `query`, returns a *sqlx.Rows instance -func (s *Session) Query(ctx context.Context, query sq.Sqlizer) (*sqlx.Rows, error) { +func (s *Session) Query(ctx context.Context, query sq.Sqlizer) (*Rows, error) { sql, args, err := s.build(query) if err != nil { return nil, err @@ -302,10 +366,26 @@ func (s *Session) Query(ctx context.Context, query sq.Sqlizer) (*sqlx.Rows, erro return s.QueryRaw(ctx, sql, args...) } +type Rows struct { + sqlx.Rows + cancel context.CancelFunc +} + +func (r *Rows) Close() error { + defer r.cancel() + return r.Rows.Close() +} + // QueryRaw runs `query` with `args` -func (s *Session) QueryRaw(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) { - query, err := s.ReplacePlaceholders(query) +func (s *Session) QueryRaw(ctx context.Context, query string, args ...interface{}) (*Rows, error) { + ctx, cancel, err := s.context(ctx) if err != nil { + return nil, err + } + + query, err = s.ReplacePlaceholders(query) + if err != nil { + cancel() return nil, errors.Wrap(err, "replace placeholders failed") } @@ -314,8 +394,12 @@ func (s *Session) QueryRaw(ctx context.Context, query string, args ...interface{ s.log(ctx, "query", start, query, args) if err == nil { - return result, nil + return &Rows{ + Rows: *result, + cancel: cancel, + }, nil } + defer cancel() if knownErr := s.handleError(err, ctx); knownErr != nil { return nil, knownErr @@ -350,6 +434,8 @@ func (s *Session) Rollback() error { log.Debug("sql: rollback") s.tx = nil s.txOptions = nil + s.txCancel() + s.txCancel = nil if knownErr := s.handleError(err, context.Background()); knownErr != nil { return knownErr @@ -381,8 +467,14 @@ func (s *Session) SelectRaw( query string, args ...interface{}, ) error { + ctx, cancel, err := s.context(ctx) + if err != nil { + return err + } + defer cancel() + s.clearSliceIfPossible(dest) - query, err := s.ReplacePlaceholders(query) + query, err = s.ReplacePlaceholders(query) if err != nil { return errors.Wrap(err, "replace placeholders failed") } diff --git a/support/db/session_test.go b/support/db/session_test.go index 1fd2a3902b..3ae06d2b4c 100644 --- a/support/db/session_test.go +++ b/support/db/session_test.go @@ -6,12 +6,12 @@ import ( "testing" "time" - //"github.com/lib/pq" "github.com/lib/pq" "github.com/prometheus/client_golang/prometheus" - "github.com/stellar/go/support/db/dbtest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/stellar/go/support/db/dbtest" ) func TestContextTimeoutDuringSql(t *testing.T) { @@ -140,6 +140,45 @@ func TestStatementTimeout(t *testing.T) { assertDbErrorMetrics(reg, "n/a", "57014", "statement_timeout", assert) } +func TestDeadlineOverride(t *testing.T) { + db := dbtest.Postgres(t).Load(testSchema) + defer db.Close() + + sess := &Session{DB: db.Open()} + defer sess.DB.Close() + + resultCtx, _, err := sess.context(context.Background()) + assert.NoError(t, err) + _, ok := resultCtx.Deadline() + assert.False(t, ok) + + deadline := time.Now().Add(time.Hour) + requestCtx := context.WithValue(context.Background(), &DeadlineCtxKey, deadline) + resultCtx, _, err = sess.context(requestCtx) + assert.NoError(t, err) + d, ok := resultCtx.Deadline() + assert.True(t, ok) + assert.Equal(t, deadline, d) + + requestCtx, cancel := context.WithDeadline(requestCtx, time.Now().Add(time.Minute*30)) + resultCtx, _, err = sess.context(requestCtx) + assert.NoError(t, err) + d, ok = resultCtx.Deadline() + assert.True(t, ok) + assert.Equal(t, deadline, d) + + cancel() + assert.NoError(t, resultCtx.Err()) + _, _, err = sess.context(requestCtx) + assert.EqualError(t, err, "context canceled") + + var emptyTime time.Time + resultCtx, _, err = sess.context(context.WithValue(context.Background(), &DeadlineCtxKey, emptyTime)) + assert.NoError(t, err) + _, ok = resultCtx.Deadline() + assert.False(t, ok) +} + func TestSession(t *testing.T) { db := dbtest.Postgres(t).Load(testSchema) defer db.Close()