From cabbe8419ee02b599f0894d7e67f0fcd8b9e2bd8 Mon Sep 17 00:00:00 2001 From: urvisavla Date: Wed, 6 Mar 2024 14:43:17 -0800 Subject: [PATCH 01/11] Bump go version to 1.22 (#5232) Co-authored-by: tamirms --- .github/workflows/go.yml | 6 +++--- .github/workflows/horizon-release.yml | 2 +- .github/workflows/horizon.yml | 4 ++-- exp/services/recoverysigner/docker/Dockerfile | 2 +- exp/services/webauth/docker/Dockerfile | 2 +- go.mod | 4 +++- go.sum | 21 +++++++++++++++++++ services/friendbot/docker/Dockerfile | 2 +- services/horizon/Makefile | 2 +- services/horizon/docker/Dockerfile.dev | 2 +- .../horizon/docker/verify-range/Dockerfile | 1 + .../horizon/docker/verify-range/dependencies | 1 - .../scripts/check_release_hash/Dockerfile | 2 +- 13 files changed, 37 insertions(+), 14 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f1fd33028d..31bd95b8ff 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: [ubuntu-22.04] - go: ["1.21"] + go: ["1.22.1"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -38,7 +38,7 @@ jobs: strategy: matrix: os: [ubuntu-22.04] - go: ["1.20", "1.21"] + go: ["1.21", "1.22"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -56,7 +56,7 @@ jobs: strategy: matrix: os: [ubuntu-22.04] - go: ["1.19", "1.20"] + go: ["1.21", "1.22"] pg: [12] runs-on: ${{ matrix.os }} services: diff --git a/.github/workflows/horizon-release.yml b/.github/workflows/horizon-release.yml index f8dda0ceac..3977ab85d3 100644 --- a/.github/workflows/horizon-release.yml +++ b/.github/workflows/horizon-release.yml @@ -22,7 +22,7 @@ jobs: - uses: ./.github/actions/setup-go with: - go-version: "1.20" + go-version: "1.22" - name: Check dependencies run: ./gomod.sh diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index bf9cae7246..3f95b9e24a 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04, ubuntu-22.04] - go: ["1.20", "1.21"] + go: ["1.21", "1.22"] pg: [12] ingestion-backend: [captive-core, captive-core-remote-storage] protocol-version: [19, 20] @@ -143,7 +143,7 @@ jobs: - name: Build and test the Verify Range Docker image run: | - docker build -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/ + docker build --build-arg="GO_VERSION=$(sed -En 's/^toolchain[[:space:]]+go([[:digit:].]+)$/\1/p' go.mod)" -f services/horizon/docker/verify-range/Dockerfile -t stellar/horizon-verify-range services/horizon/docker/verify-range/ # Any range should do for basic testing, this range was chosen pretty early in history so that it only takes a few mins to run docker run -e BRANCH=$(git rev-parse HEAD) -e FROM=10000063 -e TO=10000127 stellar/horizon-verify-range diff --git a/exp/services/recoverysigner/docker/Dockerfile b/exp/services/recoverysigner/docker/Dockerfile index 8cd9a72ae6..ff5c14e731 100644 --- a/exp/services/recoverysigner/docker/Dockerfile +++ b/exp/services/recoverysigner/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20-bullseye as build +FROM golang:1.22-bullseye as build ADD . /src/recoverysigner WORKDIR /src/recoverysigner diff --git a/exp/services/webauth/docker/Dockerfile b/exp/services/webauth/docker/Dockerfile index c6bc287d5b..64cff400aa 100644 --- a/exp/services/webauth/docker/Dockerfile +++ b/exp/services/webauth/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20-bullseye as build +FROM golang:1.22-bullseye as build ADD . /src/webauth WORKDIR /src/webauth diff --git a/go.mod b/go.mod index 0e8d89dac2..65d73fa328 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/stellar/go -go 1.20 +go 1.22 + +toolchain go1.22.1 require ( cloud.google.com/go/firestore v1.14.0 // indirect diff --git a/go.sum b/go.sum index ab7ad33230..0fe77e5201 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creachadair/jrpc2 v1.1.0 h1:SgpJf0v1rVCZx68+4APv6dgsTFsIHlpgFD1NlQAWA0A= github.com/creachadair/jrpc2 v1.1.0/go.mod h1:5jN7MKwsm8qvgfTsTzLX3JIfidsAkZ1c8DZSQmp+g38= @@ -121,6 +122,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -129,6 +131,7 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -156,10 +159,13 @@ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfC github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobuffalo/logger v1.0.6 h1:nnZNpxYo0zx+Aj9RfMPBm+x9zAU2OayFh/xrAWi34HU= +github.com/gobuffalo/logger v1.0.6/go.mod h1:J31TBEHR1QLV2683OXTAItYIg8pv2JMHnF/quuAbMjs= github.com/gobuffalo/packd v1.0.2 h1:Yg523YqnOxGIWCp69W12yYBKsoChwI7mtu6ceM9Bwfw= github.com/gobuffalo/packd v1.0.2/go.mod h1:sUc61tDqGMXON80zpKGp92lDb86Km28jfvX7IAyxFT8= github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY= +github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -217,6 +223,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -283,6 +290,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= +github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -293,6 +301,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -309,13 +318,19 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739 h1:ykXz+pRRTibcSjG1yRhpdSHInF8yZY/mfn+Rz2Nd1rE= github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739/go.mod h1:zUx1mhth20V3VKgL5jbd1BSQcW4Fy6Qs4PZvQwRFwzM= github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= +github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= +github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b/go.mod h1:5MWrJXKRQyhQdUCF+vu6U5c4nQpg70vW3eHaU0/AYbU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -332,6 +347,7 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= @@ -351,6 +367,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= +github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -470,6 +487,7 @@ go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znn go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -718,6 +736,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -826,6 +845,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/djherbis/atime.v1 v1.0.0 h1:eMRqB/JrLKocla2PBPKgQYg/p5UG4L6AUAs92aP7F60= gopkg.in/djherbis/atime.v1 v1.0.0/go.mod h1:hQIUStKmJfvf7xdh/wtK84qe+DsTV5LnA9lzxxtPpJ8= gopkg.in/djherbis/stream.v1 v1.3.1 h1:uGfmsOY1qqMjQQphhRBSGLyA9qumJ56exkRu9ASTjCw= @@ -852,6 +872,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/friendbot/docker/Dockerfile b/services/friendbot/docker/Dockerfile index dc1c74b93f..764fa5e276 100644 --- a/services/friendbot/docker/Dockerfile +++ b/services/friendbot/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20-bullseye as build +FROM golang:1.22-bullseye as build ADD . /src/friendbot WORKDIR /src/friendbot diff --git a/services/horizon/Makefile b/services/horizon/Makefile index 9c5a3a8ddf..0789453373 100644 --- a/services/horizon/Makefile +++ b/services/horizon/Makefile @@ -11,7 +11,7 @@ binary-build: --pull always \ --env CGO_ENABLED=0 \ --env GOFLAGS="-ldflags=-X=github.com/stellar/go/support/app.version=$(VERSION_STRING)" \ - golang:1.20-bullseye \ + golang:1.22-bullseye \ /bin/bash -c '\ git config --global --add safe.directory /go/src/github.com/stellar/go && \ cd /go/src/github.com/stellar/go && \ diff --git a/services/horizon/docker/Dockerfile.dev b/services/horizon/docker/Dockerfile.dev index 1d1be8d688..5cef8d89d1 100644 --- a/services/horizon/docker/Dockerfile.dev +++ b/services/horizon/docker/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.20-bullseye AS builder +FROM golang:1.22-bullseye AS builder ARG VERSION="devel" WORKDIR /go/src/github.com/stellar/go diff --git a/services/horizon/docker/verify-range/Dockerfile b/services/horizon/docker/verify-range/Dockerfile index 0143dd2cfa..6323870f38 100644 --- a/services/horizon/docker/verify-range/Dockerfile +++ b/services/horizon/docker/verify-range/Dockerfile @@ -1,5 +1,6 @@ FROM ubuntu:22.04 +ARG GO_VERSION ARG STELLAR_CORE_VERSION ENV STELLAR_CORE_VERSION=${STELLAR_CORE_VERSION:-*} # to remove tzdata interactive flow diff --git a/services/horizon/docker/verify-range/dependencies b/services/horizon/docker/verify-range/dependencies index 910ee9cf21..3eacede44b 100644 --- a/services/horizon/docker/verify-range/dependencies +++ b/services/horizon/docker/verify-range/dependencies @@ -19,7 +19,6 @@ cd stellar-go git config --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pull/*" git fetch --force --quiet origin -GO_VERSION=$(sed -En 's/^go[[:space:]]+([[:digit:].]+)$/\1/p' go.mod) wget -q https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz rm -f go${GO_VERSION}.linux-amd64.tar.gz diff --git a/services/horizon/internal/scripts/check_release_hash/Dockerfile b/services/horizon/internal/scripts/check_release_hash/Dockerfile index 6a054aab34..08f818dad3 100644 --- a/services/horizon/internal/scripts/check_release_hash/Dockerfile +++ b/services/horizon/internal/scripts/check_release_hash/Dockerfile @@ -1,5 +1,5 @@ # Change to Go version used in CI or rebuild with --build-arg. -ARG GO_IMAGE=golang:1.20-bullseye +ARG GO_IMAGE=golang:1.22-bullseye FROM $GO_IMAGE WORKDIR /go/src/github.com/stellar/go From 3fd44ca553f153840150e812fc41318abb3c2d9f Mon Sep 17 00:00:00 2001 From: shawn Date: Wed, 6 Mar 2024 15:26:13 -0800 Subject: [PATCH 02/11] services/horizon: return null txmeta in api model when SKIP_TXMETA enabled (#5228) --- clients/horizonclient/main_test.go | 35 ++++++++++- protocols/horizon/README.md | 2 + protocols/horizon/main.go | 2 +- services/horizon/CHANGELOG.md | 2 + .../horizon/internal/actions/operation.go | 8 ++- .../internal/actions/submit_transaction.go | 2 + .../horizon/internal/actions/transaction.go | 6 +- services/horizon/internal/app.go | 1 + services/horizon/internal/httpx/router.go | 20 +++++-- .../internal/integration/transaction_test.go | 17 +----- .../internal/resourceadapter/operations.go | 7 ++- .../resourceadapter/operations_test.go | 21 +++---- .../internal/resourceadapter/transaction.go | 7 ++- .../resourceadapter/transaction_test.go | 58 +++++++++++++++---- 14 files changed, 136 insertions(+), 52 deletions(-) diff --git a/clients/horizonclient/main_test.go b/clients/horizonclient/main_test.go index 2149bbbf29..f7b4ba0788 100644 --- a/clients/horizonclient/main_test.go +++ b/clients/horizonclient/main_test.go @@ -3,6 +3,7 @@ package horizonclient import ( "fmt" "net/http" + "strings" "testing" "time" @@ -905,6 +906,25 @@ func TestSubmitTransactionRequest(t *testing.T) { _, err = client.SubmitTransaction(tx) assert.NoError(t, err) + // verify submit parses correctly when result_meta_xdr absent when skip_meta=true + hmock.On( + "POST", + "https://localhost/transactions", + ).Return(func(request *http.Request) (*http.Response, error) { + val := request.FormValue("tx") + assert.Equal(t, val, txXdr) + return httpmock.NewStringResponse(http.StatusOK, strings.Replace(txDetailResponse, "", "", 1)), nil + }) + + hmock.On( + "GET", + "https://localhost/accounts/GACTJ4ZFCDZMD2UFR4R7MZOWYBCF6HBP65YKCUT37MUQFPJLDLJ3N5D2/data/config.memo_required", + ).ReturnString(404, notFoundResponse) + + theTx, err := client.SubmitTransaction(tx) + assert.NoError(t, err) + assert.Empty(t, theTx.ResultMetaXdr) + // memo required - does not submit transaction hmock.On( "GET", @@ -1388,7 +1408,7 @@ func TestTransactionsRequest(t *testing.T) { hmock.On( "GET", "https://localhost/transactions/5131aed266a639a6eb4802a92fba310454e711ded830ed899745b9e777d7110c", - ).ReturnString(200, txDetailResponse) + ).ReturnString(200, strings.Replace(txDetailResponse, "", "result_meta_xdr: AAAAAQAAAAIAAAADAAavdgAAAAAAAAAAtoYrQZHbnPLAFsF4YB88J5VSg0/piQNHm0SL9l0HW1EAAAAXSHbnnAAGr3UAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAABAAavdgAAAAAAAAAAtoYrQZHbnPLAFsF4YB88J5VSg0/piQNHm0SL9l0HW1EAAAAXSHbnnAAGr3UAAAABAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAABAAAABAAAAAMABq9zAAAAAAAAAADMSEvcRKXsaUNna++Hy7gWm/CfqTjEA7xoGypfrFGUHAAAAAUQ/z+cAABeBgAASuQAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAEABq92AAAAAAAAAADMSEvcRKXsaUNna++Hy7gWm/CfqTjEA7xoGypfrFGUHAAAAAcXjracAABeBgAASuQAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAMABq92AAAAAAAAAAC2hitBkduc8sAWwXhgHzwnlVKDT+mJA0ebRIv2XQdbUQAAABdIduecAAavdQAAAAEAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAEABq92AAAAAAAAAAC2hitBkduc8sAWwXhgHzwnlVKDT+mJA0ebRIv2XQdbUQAAABVB53CcAAavdQAAAAEAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAA==", 1)) record, err := client.TransactionDetail(txHash) if assert.NoError(t, err) { @@ -1397,6 +1417,17 @@ func TestTransactionsRequest(t *testing.T) { assert.Equal(t, record.Hash, "5131aed266a639a6eb4802a92fba310454e711ded830ed899745b9e777d7110c") assert.Equal(t, record.Memo, "2A1V6J5703G47XHY") } + + // transaction detail when skip meta enabled and result_meta_xdr is absent + hmock.On( + "GET", + "https://localhost/transactions/5131aed266a639a6eb4802a92fba310454e711ded830ed899745b9e777d7110c", + ).ReturnString(200, strings.Replace(txDetailResponse, "", "", 1)) + + record, err = client.TransactionDetail(txHash) + if assert.NoError(t, err) { + assert.Empty(t, record.ResultMetaXdr) + } } func TestOrderBookRequest(t *testing.T) { @@ -2492,7 +2523,7 @@ var txDetailResponse = `{ "operation_count": 1, "envelope_xdr": "AAAAALaGK0GR25zywBbBeGAfPCeVUoNP6YkDR5tEi/ZdB1tRAAAAZAAGr3UAAAABAAAAAAAAAAEAAAAQMkExVjZKNTcwM0c0N1hIWQAAAAEAAAABAAAAALaGK0GR25zywBbBeGAfPCeVUoNP6YkDR5tEi/ZdB1tRAAAAAQAAAADMSEvcRKXsaUNna++Hy7gWm/CfqTjEA7xoGypfrFGUHAAAAAAAAAACBo93AAAAAAAAAAABXQdbUQAAAECQ5m6ZHsv8/Gd/aRJ2EMLurJMxFynT7KbD51T7gD91Gqp/fzsRHilSGoVSw5ztmtJb2LP7o3bQbiZynQiJPl8C", "result_xdr": "AAAAAAAAAGQAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAA=", - "result_meta_xdr": "AAAAAQAAAAIAAAADAAavdgAAAAAAAAAAtoYrQZHbnPLAFsF4YB88J5VSg0/piQNHm0SL9l0HW1EAAAAXSHbnnAAGr3UAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAABAAavdgAAAAAAAAAAtoYrQZHbnPLAFsF4YB88J5VSg0/piQNHm0SL9l0HW1EAAAAXSHbnnAAGr3UAAAABAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAABAAAABAAAAAMABq9zAAAAAAAAAADMSEvcRKXsaUNna++Hy7gWm/CfqTjEA7xoGypfrFGUHAAAAAUQ/z+cAABeBgAASuQAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAEABq92AAAAAAAAAADMSEvcRKXsaUNna++Hy7gWm/CfqTjEA7xoGypfrFGUHAAAAAcXjracAABeBgAASuQAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAMABq92AAAAAAAAAAC2hitBkduc8sAWwXhgHzwnlVKDT+mJA0ebRIv2XQdbUQAAABdIduecAAavdQAAAAEAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAEABq92AAAAAAAAAAC2hitBkduc8sAWwXhgHzwnlVKDT+mJA0ebRIv2XQdbUQAAABVB53CcAAavdQAAAAEAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAA==", + "result_meta_xdr": "", "fee_meta_xdr": "AAAAAgAAAAMABq91AAAAAAAAAAC2hitBkduc8sAWwXhgHzwnlVKDT+mJA0ebRIv2XQdbUQAAABdIdugAAAavdQAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAEABq92AAAAAAAAAAC2hitBkduc8sAWwXhgHzwnlVKDT+mJA0ebRIv2XQdbUQAAABdIduecAAavdQAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAA==", "memo_type": "text", "signatures": [ diff --git a/protocols/horizon/README.md b/protocols/horizon/README.md index c6f7ba2654..54b9ed1449 100644 --- a/protocols/horizon/README.md +++ b/protocols/horizon/README.md @@ -17,6 +17,8 @@ For each new version we will only track changes from the previous version. #### Changes +* In ["Transaction"](https://developers.stellar.org/api/horizon/resources/transactions/object), +`result_meta_xdr` field is [now nullable](https://github.com/stellar/go/pull/5228), and will be `null` when Horizon has `SKIP_TXMETA=true` set, otherwise if Horizon is configured with `SKIP_TXMETA=false` which is default, then `result_meta_xdr` will be the same value of base64 encoded xdr. * Operations responses may include a `transaction` field which represents the transaction that created the operation. ### 0.15.0 diff --git a/protocols/horizon/main.go b/protocols/horizon/main.go index 08da47b7b4..bdec98ba0b 100644 --- a/protocols/horizon/main.go +++ b/protocols/horizon/main.go @@ -518,7 +518,7 @@ type Transaction struct { OperationCount int32 `json:"operation_count"` EnvelopeXdr string `json:"envelope_xdr"` ResultXdr string `json:"result_xdr"` - ResultMetaXdr string `json:"result_meta_xdr"` + ResultMetaXdr string `json:"result_meta_xdr,omitempty"` FeeMetaXdr string `json:"fee_meta_xdr"` MemoType string `json:"memo_type"` MemoBytes string `json:"memo_bytes,omitempty"` diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 7bf15774f1..b036851710 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -11,6 +11,8 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed - History archive access is more effective when you pass list of URLs to Horizon: they will now be accessed in a round-robin fashion, use alternative archives on errors, and intelligently back off ([5224](https://github.com/stellar/go/pull/5224)) +### Breaking Changes +- The Horizon API Transaction resource field in json `result_meta_xdr` is now optional and Horizon API will not emit the field when Horizon has been configured with `SKIP_TXMETA=true`, effectively null, otherwise if Horizon is configured with `SKIP_TXMETA=false` which is default, then the API Transaction field `result_meta_xdr` will remain present and populated with base64 encoded xdr [5228](https://github.com/stellar/go/pull/5228). ## 2.28.3 diff --git a/services/horizon/internal/actions/operation.go b/services/horizon/internal/actions/operation.go index 670f592b57..f59191aee0 100644 --- a/services/horizon/internal/actions/operation.go +++ b/services/horizon/internal/actions/operation.go @@ -65,6 +65,7 @@ func (qp OperationsQuery) Validate() error { type GetOperationsHandler struct { LedgerState *ledger.State OnlyPayments bool + SkipTxMeta bool } // GetResourcePage returns a page of operations. @@ -126,12 +127,13 @@ func (handler GetOperationsHandler) GetResourcePage(w HeaderWriter, r *http.Requ return nil, err } - return buildOperationsPage(ctx, historyQ, ops, txs, qp.IncludeTransactions()) + return buildOperationsPage(ctx, historyQ, ops, txs, qp.IncludeTransactions(), handler.SkipTxMeta) } // GetOperationByIDHandler is the action handler for all end-points returning a list of operations. type GetOperationByIDHandler struct { LedgerState *ledger.State + SkipTxMeta bool } // OperationQuery query struct for operation/id end-point @@ -182,10 +184,11 @@ func (handler GetOperationByIDHandler) GetResource(w HeaderWriter, r *http.Reque op.TransactionHash, tx, ledger, + handler.SkipTxMeta, ) } -func buildOperationsPage(ctx context.Context, historyQ *history.Q, operations []history.Operation, transactions []history.Transaction, includeTransactions bool) ([]hal.Pageable, error) { +func buildOperationsPage(ctx context.Context, historyQ *history.Q, operations []history.Operation, transactions []history.Transaction, includeTransactions bool, skipTxMeta bool) ([]hal.Pageable, error) { ledgerCache := history.LedgerCache{} for _, record := range operations { ledgerCache.Queue(record.LedgerSequence()) @@ -216,6 +219,7 @@ func buildOperationsPage(ctx context.Context, historyQ *history.Q, operations [] operationRecord.TransactionHash, transactionRecord, ledger, + skipTxMeta, ) if err != nil { return nil, err diff --git a/services/horizon/internal/actions/submit_transaction.go b/services/horizon/internal/actions/submit_transaction.go index b877f75a7b..314caf32a5 100644 --- a/services/horizon/internal/actions/submit_transaction.go +++ b/services/horizon/internal/actions/submit_transaction.go @@ -27,6 +27,7 @@ type SubmitTransactionHandler struct { NetworkPassphrase string DisableTxSub bool CoreStateGetter + SkipTxMeta bool } type envelopeInfo struct { @@ -84,6 +85,7 @@ func (handler SubmitTransactionHandler) response(r *http.Request, info envelopeI info.hash, &resource, result.Transaction, + handler.SkipTxMeta, ) return resource, err } diff --git a/services/horizon/internal/actions/transaction.go b/services/horizon/internal/actions/transaction.go index f823ad9acb..6903d5db2f 100644 --- a/services/horizon/internal/actions/transaction.go +++ b/services/horizon/internal/actions/transaction.go @@ -23,6 +23,7 @@ type TransactionQuery struct { // GetTransactionByHashHandler is the action handler for the end-point returning a transaction. type GetTransactionByHashHandler struct { + SkipTxMeta bool } // GetResource returns a transaction page. @@ -49,7 +50,7 @@ func (handler GetTransactionByHashHandler) GetResource(w HeaderWriter, r *http.R return resource, errors.Wrap(err, "loading transaction record") } - if err = resourceadapter.PopulateTransaction(ctx, qp.TransactionHash, &resource, record); err != nil { + if err = resourceadapter.PopulateTransaction(ctx, qp.TransactionHash, &resource, record, handler.SkipTxMeta); err != nil { return resource, errors.Wrap(err, "could not populate transaction") } return resource, nil @@ -90,6 +91,7 @@ func (qp TransactionsQuery) Validate() error { // GetTransactionsHandler is the action handler for all end-points returning a list of transactions. type GetTransactionsHandler struct { LedgerState *ledger.State + SkipTxMeta bool } // GetResourcePage returns a page of transactions. @@ -126,7 +128,7 @@ func (handler GetTransactionsHandler) GetResourcePage(w HeaderWriter, r *http.Re for _, record := range records { var res horizon.Transaction - err = resourceadapter.PopulateTransaction(ctx, record.TransactionHash, &res, record) + err = resourceadapter.PopulateTransaction(ctx, record.TransactionHash, &res, record, handler.SkipTxMeta) if err != nil { return nil, errors.Wrap(err, "could not populate transaction") } diff --git a/services/horizon/internal/app.go b/services/horizon/internal/app.go index b1cd7a1c85..59a8fb4432 100644 --- a/services/horizon/internal/app.go +++ b/services/horizon/internal/app.go @@ -552,6 +552,7 @@ func (a *App) init() error { }, cache: newHealthCache(healthCacheTTL), }, + SkipTxMeta: a.config.SkipTxmeta, } if a.primaryHistoryQ != nil { diff --git a/services/horizon/internal/httpx/router.go b/services/horizon/internal/httpx/router.go index 8fa57d0379..2755b9e062 100644 --- a/services/horizon/internal/httpx/router.go +++ b/services/horizon/internal/httpx/router.go @@ -50,6 +50,7 @@ type RouterConfig struct { HealthCheck http.Handler EnableIngestionFiltering bool DisableTxSub bool + SkipTxMeta bool } type Router struct { @@ -191,8 +192,9 @@ func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRate r.With(historyMiddleware).Method(http.MethodGet, "/operations", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: false, + SkipTxMeta: config.SkipTxMeta, }, streamHandler)) - r.With(historyMiddleware).Method(http.MethodGet, "/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState}, streamHandler)) + r.With(historyMiddleware).Method(http.MethodGet, "/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState, SkipTxMeta: config.SkipTxMeta}, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/effects", streamableHistoryPageHandler(ledgerState, actions.GetEffectsHandler{LedgerState: ledgerState}, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/trades", streamableHistoryPageHandler(ledgerState, actions.GetTradesHandler{LedgerState: ledgerState, CoreStateGetter: config.CoreGetter}, streamHandler)) }) @@ -241,29 +243,32 @@ func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRate r.With(historyMiddleware).Method(http.MethodGet, "/accounts/{account_id:\\w+}/operations", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: false, + SkipTxMeta: config.SkipTxMeta, }, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/accounts/{account_id:\\w+}/payments", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: true, }, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/accounts/{account_id:\\w+}/trades", streamableHistoryPageHandler(ledgerState, actions.GetTradesHandler{LedgerState: ledgerState, CoreStateGetter: config.CoreGetter}, streamHandler)) - r.With(historyMiddleware).Method(http.MethodGet, "/accounts/{account_id:\\w+}/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState}, streamHandler)) + r.With(historyMiddleware).Method(http.MethodGet, "/accounts/{account_id:\\w+}/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState, SkipTxMeta: config.SkipTxMeta}, streamHandler)) }) // ledger actions r.Route("/ledgers", func(r chi.Router) { r.With(historyMiddleware).Method(http.MethodGet, "/", streamableHistoryPageHandler(ledgerState, actions.GetLedgersHandler{LedgerState: ledgerState}, streamHandler)) r.Route("/{ledger_id}", func(r chi.Router) { r.With(historyMiddleware).Method(http.MethodGet, "/", ObjectActionHandler{actions.GetLedgerByIDHandler{LedgerState: ledgerState}}) - r.With(historyMiddleware).Method(http.MethodGet, "/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState}, streamHandler)) + r.With(historyMiddleware).Method(http.MethodGet, "/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState, SkipTxMeta: config.SkipTxMeta}, streamHandler)) r.Group(func(r chi.Router) { r.With(historyMiddleware).Method(http.MethodGet, "/effects", streamableHistoryPageHandler(ledgerState, actions.GetEffectsHandler{LedgerState: ledgerState}, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/operations", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: false, + SkipTxMeta: config.SkipTxMeta, }, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/payments", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: true, + SkipTxMeta: config.SkipTxMeta, }, streamHandler)) }) }) @@ -275,18 +280,19 @@ func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRate LedgerState: ledgerState, OnlyPayments: false, }, streamHandler)) - r.With(historyMiddleware).Method(http.MethodGet, "/claimable_balances/{claimable_balance_id:\\w+}/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState}, streamHandler)) + r.With(historyMiddleware).Method(http.MethodGet, "/claimable_balances/{claimable_balance_id:\\w+}/transactions", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState, SkipTxMeta: config.SkipTxMeta}, streamHandler)) }) // transaction history actions r.Route("/transactions", func(r chi.Router) { - r.With(historyMiddleware).Method(http.MethodGet, "/", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState}, streamHandler)) + r.With(historyMiddleware).Method(http.MethodGet, "/", streamableHistoryPageHandler(ledgerState, actions.GetTransactionsHandler{LedgerState: ledgerState, SkipTxMeta: config.SkipTxMeta}, streamHandler)) r.Route("/{tx_id}", func(r chi.Router) { r.With(historyMiddleware).Method(http.MethodGet, "/", ObjectActionHandler{actions.GetTransactionByHashHandler{}}) r.With(historyMiddleware).Method(http.MethodGet, "/effects", streamableHistoryPageHandler(ledgerState, actions.GetEffectsHandler{LedgerState: ledgerState}, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/operations", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: false, + SkipTxMeta: config.SkipTxMeta, }, streamHandler)) r.With(historyMiddleware).Method(http.MethodGet, "/payments", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, @@ -300,8 +306,9 @@ func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRate r.With(historyMiddleware).Method(http.MethodGet, "/", streamableHistoryPageHandler(ledgerState, actions.GetOperationsHandler{ LedgerState: ledgerState, OnlyPayments: false, + SkipTxMeta: config.SkipTxMeta, }, streamHandler)) - r.With(historyMiddleware).Method(http.MethodGet, "/{id}", ObjectActionHandler{actions.GetOperationByIDHandler{LedgerState: ledgerState}}) + r.With(historyMiddleware).Method(http.MethodGet, "/{id}", ObjectActionHandler{actions.GetOperationByIDHandler{LedgerState: ledgerState, SkipTxMeta: config.SkipTxMeta}}) r.With(historyMiddleware).Method(http.MethodGet, "/{op_id}/effects", streamableHistoryPageHandler(ledgerState, actions.GetEffectsHandler{LedgerState: ledgerState}, streamHandler)) }) @@ -329,6 +336,7 @@ func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRate NetworkPassphrase: config.NetworkPassphrase, DisableTxSub: config.DisableTxSub, CoreStateGetter: config.CoreGetter, + SkipTxMeta: config.SkipTxMeta, }}) // Network state related endpoints diff --git a/services/horizon/internal/integration/transaction_test.go b/services/horizon/internal/integration/transaction_test.go index 85fbe78522..a0db5816dd 100644 --- a/services/horizon/internal/integration/transaction_test.go +++ b/services/horizon/internal/integration/transaction_test.go @@ -63,13 +63,7 @@ func TestP19MetaDisabledTransaction(t *testing.T) { clientTx := itest.MustSubmitOperations(&masterAccount, itest.Master(), op) - var txMetaResult xdr.TransactionMeta - err = xdr.SafeUnmarshalBase64(clientTx.ResultMetaXdr, &txMetaResult) - require.NoError(t, err) - - assert.Equal(t, len(txMetaResult.MustV2().Operations), 0) - assert.Equal(t, len(txMetaResult.MustV2().TxChangesAfter), 0) - assert.Equal(t, len(txMetaResult.MustV2().TxChangesBefore), 0) + assert.Empty(t, clientTx.ResultMetaXdr) } func TestP20MetaTransaction(t *testing.T) { @@ -123,12 +117,5 @@ func TestP20MetaDisabledTransaction(t *testing.T) { preFlightOp, minFee := itest.PreflightHostFunctions(&sourceAccount, *installContractOp) clientTx := itest.MustSubmitOperationsWithFee(&sourceAccount, itest.Master(), minFee+txnbuild.MinBaseFee, &preFlightOp) - var txMetaResult xdr.TransactionMeta - err = xdr.SafeUnmarshalBase64(clientTx.ResultMetaXdr, &txMetaResult) - require.NoError(t, err) - - assert.Equal(t, len(txMetaResult.MustV3().Operations), 0) - assert.Nil(t, txMetaResult.MustV3().SorobanMeta) - assert.Equal(t, len(txMetaResult.MustV3().TxChangesAfter), 0) - assert.Equal(t, len(txMetaResult.MustV3().TxChangesBefore), 0) + assert.Empty(t, clientTx.ResultMetaXdr) } diff --git a/services/horizon/internal/resourceadapter/operations.go b/services/horizon/internal/resourceadapter/operations.go index 2f995fb395..fc301aec64 100644 --- a/services/horizon/internal/resourceadapter/operations.go +++ b/services/horizon/internal/resourceadapter/operations.go @@ -20,10 +20,11 @@ func NewOperation( transactionHash string, transactionRow *history.Transaction, ledger history.Ledger, + skipTxMeta bool, ) (result hal.Pageable, err error) { base := operations.Base{} - err = PopulateBaseOperation(ctx, &base, operationRow, transactionHash, transactionRow, ledger) + err = PopulateBaseOperation(ctx, &base, operationRow, transactionHash, transactionRow, ledger, skipTxMeta) if err != nil { return } @@ -166,7 +167,7 @@ func NewOperation( } // Populate fills out this resource using `row` as the source. -func PopulateBaseOperation(ctx context.Context, dest *operations.Base, operationRow history.Operation, transactionHash string, transactionRow *history.Transaction, ledger history.Ledger) error { +func PopulateBaseOperation(ctx context.Context, dest *operations.Base, operationRow history.Operation, transactionHash string, transactionRow *history.Transaction, ledger history.Ledger, skipTxMeta bool) error { dest.ID = fmt.Sprintf("%d", operationRow.ID) dest.PT = operationRow.PagingToken() dest.TransactionSuccessful = operationRow.TransactionSuccessful @@ -190,7 +191,7 @@ func PopulateBaseOperation(ctx context.Context, dest *operations.Base, operation if transactionRow != nil { dest.Transaction = new(horizon.Transaction) - return PopulateTransaction(ctx, transactionHash, dest.Transaction, *transactionRow) + return PopulateTransaction(ctx, transactionHash, dest.Transaction, *transactionRow, skipTxMeta) } return nil } diff --git a/services/horizon/internal/resourceadapter/operations_test.go b/services/horizon/internal/resourceadapter/operations_test.go index 39660a3678..f6acfa664b 100644 --- a/services/horizon/internal/resourceadapter/operations_test.go +++ b/services/horizon/internal/resourceadapter/operations_test.go @@ -19,7 +19,7 @@ func TestNewOperationAllTypesCovered(t *testing.T) { row := history.Operation{ Type: xdr.OperationType(typ), } - op, err := NewOperation(context.Background(), row, "foo", tx, history.Ledger{}) + op, err := NewOperation(context.Background(), row, "foo", tx, history.Ledger{}, false) assert.NoError(t, err, s) // if we got a base type, the operation is not covered if _, ok := op.(operations.Base); ok { @@ -31,7 +31,7 @@ func TestNewOperationAllTypesCovered(t *testing.T) { row := history.Operation{ Type: xdr.OperationType(200000), } - op, err := NewOperation(context.Background(), row, "foo", tx, history.Ledger{}) + op, err := NewOperation(context.Background(), row, "foo", tx, history.Ledger{}, false) assert.NoError(t, err) assert.IsType(t, op, operations.Base{}) @@ -52,7 +52,7 @@ func TestPopulateOperation_Successful(t *testing.T) { assert.NoError( t, - PopulateBaseOperation(ctx, &dest, row, "", nil, ledger), + PopulateBaseOperation(ctx, &dest, row, "", nil, ledger, false), ) assert.True(t, dest.TransactionSuccessful) assert.Nil(t, dest.Transaction) @@ -62,7 +62,7 @@ func TestPopulateOperation_Successful(t *testing.T) { assert.NoError( t, - PopulateBaseOperation(ctx, &dest, row, "", nil, ledger), + PopulateBaseOperation(ctx, &dest, row, "", nil, ledger, false), ) assert.False(t, dest.TransactionSuccessful) assert.Nil(t, dest.Transaction) @@ -92,12 +92,13 @@ func TestPopulateOperation_WithTransaction(t *testing.T) { assert.NoError( t, - PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.TransactionHash, &transactionRow, ledger), + PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.TransactionHash, &transactionRow, ledger, true), ) assert.True(t, dest.TransactionSuccessful) assert.True(t, dest.Transaction.Successful) assert.Equal(t, int64(100), dest.Transaction.FeeCharged) assert.Equal(t, int64(10000), dest.Transaction.MaxFee) + assert.Empty(t, dest.Transaction.ResultMetaXdr) } func TestPopulateOperation_AllowTrust(t *testing.T) { @@ -308,7 +309,7 @@ func getJSONResponse(typ xdr.OperationType, details string) (rsp map[string]inte Type: typ, DetailsString: null.StringFrom(details), } - resource, err := NewOperation(ctx, operationsRow, "", &transactionRow, history.Ledger{}) + resource, err := NewOperation(ctx, operationsRow, "", &transactionRow, history.Ledger{}, false) if err != nil { return } @@ -343,19 +344,19 @@ func TestFeeBumpOperation(t *testing.T) { assert.NoError( t, - PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.TransactionHash, nil, history.Ledger{}), + PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.TransactionHash, nil, history.Ledger{}, false), ) assert.Equal(t, transactionRow.TransactionHash, dest.TransactionHash) assert.NoError( t, - PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.InnerTransactionHash.String, nil, history.Ledger{}), + PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.InnerTransactionHash.String, nil, history.Ledger{}, false), ) assert.Equal(t, transactionRow.InnerTransactionHash.String, dest.TransactionHash) assert.NoError( t, - PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.TransactionHash, &transactionRow, history.Ledger{}), + PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.TransactionHash, &transactionRow, history.Ledger{}, false), ) assert.Equal(t, transactionRow.TransactionHash, dest.TransactionHash) @@ -374,7 +375,7 @@ func TestFeeBumpOperation(t *testing.T) { assert.NoError( t, - PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.InnerTransactionHash.String, &transactionRow, history.Ledger{}), + PopulateBaseOperation(ctx, &dest, operationsRow, transactionRow.InnerTransactionHash.String, &transactionRow, history.Ledger{}, false), ) assert.Equal(t, transactionRow.InnerTransactionHash.String, dest.TransactionHash) assert.Equal(t, transactionRow.InnerTransactionHash.String, dest.Transaction.Hash) diff --git a/services/horizon/internal/resourceadapter/transaction.go b/services/horizon/internal/resourceadapter/transaction.go index 547f8f4b40..f458830c2d 100644 --- a/services/horizon/internal/resourceadapter/transaction.go +++ b/services/horizon/internal/resourceadapter/transaction.go @@ -23,6 +23,7 @@ func PopulateTransaction( transactionHash string, dest *protocol.Transaction, row history.Transaction, + skipTxMeta bool, ) error { dest.ID = transactionHash dest.PT = row.PagingToken() @@ -43,7 +44,11 @@ func PopulateTransaction( dest.OperationCount = row.OperationCount dest.EnvelopeXdr = row.TxEnvelope dest.ResultXdr = row.TxResult - dest.ResultMetaXdr = row.TxMeta + if skipTxMeta { + dest.ResultMetaXdr = "" + } else { + dest.ResultMetaXdr = row.TxMeta + } dest.FeeMetaXdr = row.TxFeeMeta dest.MemoType = row.MemoType dest.Memo = row.Memo.String diff --git a/services/horizon/internal/resourceadapter/transaction_test.go b/services/horizon/internal/resourceadapter/transaction_test.go index 29c8040ce6..694fc885fb 100644 --- a/services/horizon/internal/resourceadapter/transaction_test.go +++ b/services/horizon/internal/resourceadapter/transaction_test.go @@ -34,18 +34,40 @@ func TestPopulateTransaction_Successful(t *testing.T) { }, } - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) assert.True(t, dest.Successful) dest = Transaction{} row = history.Transaction{ TransactionWithoutLedger: history.TransactionWithoutLedger{ Successful: false, + TxMeta: "xyz", }, } - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) assert.False(t, dest.Successful) + assert.NotEmpty(t, dest.ResultMetaXdr) +} + +func TestPopulateTransactionWhenSkipMeta(t *testing.T) { + ctx, _ := test.ContextWithLogBuffer() + + var ( + dest Transaction + row history.Transaction + ) + + dest = Transaction{} + row = history.Transaction{ + TransactionWithoutLedger: history.TransactionWithoutLedger{ + Successful: true, + }, + } + + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, true)) + assert.True(t, dest.Successful) + assert.Empty(t, dest.ResultMetaXdr) } func TestPopulateTransaction_HashMemo(t *testing.T) { @@ -55,12 +77,14 @@ func TestPopulateTransaction_HashMemo(t *testing.T) { TransactionWithoutLedger: history.TransactionWithoutLedger{ MemoType: "hash", Memo: null.StringFrom("abcdef"), + TxMeta: "xyz", }, } - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) assert.Equal(t, "hash", dest.MemoType) assert.Equal(t, "abcdef", dest.Memo) assert.Equal(t, "", dest.MemoBytes) + assert.NotEmpty(t, dest.ResultMetaXdr) } func TestPopulateTransaction_TextMemo(t *testing.T) { @@ -122,15 +146,17 @@ func TestPopulateTransaction_TextMemo(t *testing.T) { MemoType: "text", TxEnvelope: envelopeXDR, Memo: null.StringFrom("sample"), + TxMeta: "xyz", }, } var dest Transaction - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) assert.Equal(t, "text", dest.MemoType) assert.Equal(t, "sample", dest.Memo) assert.Equal(t, base64.StdEncoding.EncodeToString(rawMemo), dest.MemoBytes) + assert.NotEmpty(t, dest.ResultMetaXdr) } } @@ -148,12 +174,14 @@ func TestPopulateTransaction_Fee(t *testing.T) { TransactionWithoutLedger: history.TransactionWithoutLedger{ MaxFee: 10000, FeeCharged: 100, + TxMeta: "xyz", }, } - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) assert.Equal(t, int64(100), dest.FeeCharged) assert.Equal(t, int64(10000), dest.MaxFee) + assert.NotEmpty(t, dest.ResultMetaXdr) } // TestPopulateTransaction_Preconditions tests transaction object population. @@ -188,10 +216,12 @@ func TestPopulateTransaction_Preconditions(t *testing.T) { MinAccountSequenceAge: null.StringFrom(fmt.Sprint(minSequenceAge)), MinAccountSequenceLedgerGap: null.IntFrom(int64(minSequenceLedgerGap)), ExtraSigners: pq.StringArray{"D34DB33F", "8BADF00D"}, + TxMeta: "xyz", }, } - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) + assert.NotEmpty(t, dest.ResultMetaXdr) p := dest.Preconditions assert.Equal(t, validAfter.Format(time.RFC3339), dest.ValidAfter) assert.Equal(t, validBefore.Format(time.RFC3339), dest.ValidBefore) @@ -282,11 +312,13 @@ func TestPopulateTransaction_PreconditionsV2(t *testing.T) { Upper: null.IntFrom(int64(envelopeTimebounds.MaxTime)), }, TxEnvelope: envelopeXDR, + TxMeta: "xyz", }, } var dest Transaction - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) + assert.NotEmpty(t, dest.ResultMetaXdr) gotTimebounds := dest.Preconditions.TimeBounds assert.Equal(t, "5", gotTimebounds.MinTime) @@ -307,7 +339,7 @@ func TestPopulateTransaction_PreconditionsV2_Omissions(t *testing.T) { generic := map[string]interface{}{} row := history.Transaction{TransactionWithoutLedger: tx} - tt.NoError(PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + tt.NoError(PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) bytes, err := dest.MarshalJSON() tt.NoError(err) @@ -325,6 +357,7 @@ func TestPopulateTransaction_PreconditionsV2_Omissions(t *testing.T) { MinAccountSequence: null.IntFromPtr(nil), MinAccountSequenceAge: null.StringFrom("0"), ExtraSigners: pq.StringArray{}, + TxMeta: "xyz", }, { AccountSequence: 1, MinAccountSequenceLedgerGap: null.IntFrom(0), @@ -333,6 +366,7 @@ func TestPopulateTransaction_PreconditionsV2_Omissions(t *testing.T) { MinAccountSequence: null.IntFromPtr(nil), MinAccountSequenceAge: null.StringFromPtr(nil), ExtraSigners: nil, + TxMeta: "xyz", }, } { dest, js := jsonifyTx(tx) @@ -354,6 +388,7 @@ func TestPopulateTransaction_PreconditionsV2_Omissions(t *testing.T) { // exist entirely. tx.MinAccountSequenceLedgerGap = null.IntFromPtr(nil) dest, js = jsonifyTx(tx) + assert.NotEmpty(t, dest.ResultMetaXdr) tt.NotContains(js, "preconditions") } } @@ -375,10 +410,12 @@ func TestFeeBumpTransaction(t *testing.T) { InnerTransactionHash: null.StringFrom("2374e99349b9ef7dba9a5db3339b78fda8f34777b1af33ba468ad5c0df946d4d"), Signatures: []string{"a", "b", "c"}, InnerSignatures: []string{"d", "e", "f"}, + TxMeta: "xyz", }, } - assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.TransactionHash, &dest, row, false)) + assert.NotEmpty(t, dest.ResultMetaXdr) assert.Equal(t, row.TransactionHash, dest.Hash) assert.Equal(t, row.TransactionHash, dest.ID) assert.Equal(t, row.FeeAccount.String, dest.FeeAccount) @@ -397,7 +434,8 @@ func TestFeeBumpTransaction(t *testing.T) { assert.Equal(t, []string{"a", "b", "c"}, dest.FeeBumpTransaction.Signatures) assert.Equal(t, "/transactions/"+row.TransactionHash, dest.Links.Transaction.Href) - assert.NoError(t, PopulateTransaction(ctx, row.InnerTransactionHash.String, &dest, row)) + assert.NoError(t, PopulateTransaction(ctx, row.InnerTransactionHash.String, &dest, row, false)) + assert.NotEmpty(t, dest.ResultMetaXdr) assert.Equal(t, row.InnerTransactionHash.String, dest.Hash) assert.Equal(t, row.InnerTransactionHash.String, dest.ID) assert.Equal(t, row.FeeAccount.String, dest.FeeAccount) From 73c6f37751eec0a78a765e665c32b55540fb5492 Mon Sep 17 00:00:00 2001 From: urvisavla Date: Thu, 7 Mar 2024 11:53:57 -0800 Subject: [PATCH 03/11] services/horizon: Update CHANGELOG (#5238) --- services/horizon/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index b036851710..4003073503 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -7,6 +7,7 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). ### Added - New `db_error_total` metrics key with labels `ctx_error`, `db_error`, and `db_error_extra` ([5225](https://github.com/stellar/go/pull/5225)). +- Bumped go version to the latest (1.22.1) ### Fixed - History archive access is more effective when you pass list of URLs to Horizon: they will now be accessed in a round-robin fashion, use alternative archives on errors, and intelligently back off ([5224](https://github.com/stellar/go/pull/5224)) From 324ea1a238ab34cb87dc427e59a0c2619a5d4b8c Mon Sep 17 00:00:00 2001 From: tamirms Date: Mon, 11 Mar 2024 15:59:51 +0000 Subject: [PATCH 04/11] simplify effects query paging where clause (#5233) --- services/horizon/internal/actions/effects.go | 19 +- .../horizon/internal/db2/history/effect.go | 165 +++++++++--------- .../effect_batch_insert_builder_test.go | 9 +- .../internal/db2/history/effect_test.go | 12 +- services/horizon/internal/db2/history/main.go | 8 - .../internal/db2/history/transaction_test.go | 14 +- 6 files changed, 119 insertions(+), 108 deletions(-) diff --git a/services/horizon/internal/actions/effects.go b/services/horizon/internal/actions/effects.go index a141067d25..cc7406d7bc 100644 --- a/services/horizon/internal/actions/effects.go +++ b/services/horizon/internal/actions/effects.go @@ -95,25 +95,20 @@ func (handler GetEffectsHandler) GetResourcePage(w HeaderWriter, r *http.Request } func loadEffectRecords(ctx context.Context, hq *history.Q, qp EffectsQuery, pq db2.PageQuery) ([]history.Effect, error) { - effects := hq.Effects() - switch { case qp.AccountID != "": - effects.ForAccount(ctx, qp.AccountID) + return hq.EffectsForAccount(ctx, qp.AccountID, pq) case qp.LiquidityPoolID != "": - effects.ForLiquidityPool(ctx, pq, qp.LiquidityPoolID) + return hq.EffectsForLiquidityPool(ctx, qp.LiquidityPoolID, pq) case qp.OperationID > 0: - effects.ForOperation(int64(qp.OperationID)) + return hq.EffectsForOperation(ctx, int64(qp.OperationID), pq) case qp.LedgerID > 0: - effects.ForLedger(ctx, int32(qp.LedgerID)) + return hq.EffectsForLedger(ctx, int32(qp.LedgerID), pq) case qp.TxHash != "": - effects.ForTransaction(ctx, qp.TxHash) + return hq.EffectsForTransaction(ctx, qp.TxHash, pq) + default: + return hq.Effects(ctx, pq) } - - var result []history.Effect - err := effects.Page(pq).Select(ctx, &result) - - return result, err } func loadEffectLedgers(ctx context.Context, hq *history.Q, effects []history.Effect) (map[int32]history.Ledger, error) { diff --git a/services/horizon/internal/db2/history/effect.go b/services/horizon/internal/db2/history/effect.go index 13a9c52519..bdf1e2dfb0 100644 --- a/services/horizon/internal/db2/history/effect.go +++ b/services/horizon/internal/db2/history/effect.go @@ -69,75 +69,83 @@ func (r *Effect) PagingToken() string { return fmt.Sprintf("%d-%d", r.HistoryOperationID, r.Order) } -// Effects provides a helper to filter rows from the `history_effects` -// table with pre-defined filters. See `TransactionsQ` methods for the -// available filters. -func (q *Q) Effects() *EffectsQ { - return &EffectsQ{ - parent: q, - sql: selectEffect, +// Effects returns a page of effects without any filters besides the cursor +func (q *Q) Effects(ctx context.Context, page db2.PageQuery) ([]Effect, error) { + op, idx, err := parseEffectsCursor(page) + if err != nil { + return nil, err + } + + var rows []Effect + query := selectEffect + // we do not use selectEffectsPage() because we have found the + // query below to be more efficient when there are no other constraints + // such as filtering by account / ledger / transaction / etc + switch page.Order { + case "asc": + query = query. + Where("(heff.history_operation_id, heff.order) > (?, ?)", op, idx). + OrderBy("heff.history_operation_id asc, heff.order asc") + case "desc": + query = query. + Where("(heff.history_operation_id, heff.order) < (?, ?)", op, idx). + OrderBy("heff.history_operation_id desc, heff.order desc") } + + query = query.Limit(page.Limit) + + if err = q.Select(ctx, &rows, query); err != nil { + return nil, err + } + return rows, nil } -// ForAccount filters the operations collection to a specific account -func (q *EffectsQ) ForAccount(ctx context.Context, aid string) *EffectsQ { +// EffectsForAccount returns a page of effects for a given account +func (q *Q) EffectsForAccount(ctx context.Context, aid string, page db2.PageQuery) ([]Effect, error) { var account Account - q.Err = q.parent.AccountByAddress(ctx, &account, aid) - if q.Err != nil { - return q + if err := q.AccountByAddress(ctx, &account, aid); err != nil { + return nil, err } - q.sql = q.sql.Where("heff.history_account_id = ?", account.ID) - - return q + query := selectEffect.Where("heff.history_account_id = ?", account.ID) + return q.selectEffectsPage(ctx, query, page) } -// ForLedger filters the query to only effects in a specific ledger, -// specified by its sequence. -func (q *EffectsQ) ForLedger(ctx context.Context, seq int32) *EffectsQ { +// EffectsForLedger returns a page of effects for a given ledger sequence +func (q *Q) EffectsForLedger(ctx context.Context, seq int32, page db2.PageQuery) ([]Effect, error) { var ledger Ledger - q.Err = q.parent.LedgerBySequence(ctx, &ledger, seq) - if q.Err != nil { - return q + if err := q.LedgerBySequence(ctx, &ledger, seq); err != nil { + return nil, err } start := toid.ID{LedgerSequence: seq} end := toid.ID{LedgerSequence: seq + 1} - q.sql = q.sql.Where( + query := selectEffect.Where( "heff.history_operation_id >= ? AND heff.history_operation_id < ?", start.ToInt64(), end.ToInt64(), ) - - return q + return q.selectEffectsPage(ctx, query, page) } -// ForOperation filters the query to only effects in a specific operation, -// specified by its id. -func (q *EffectsQ) ForOperation(id int64) *EffectsQ { +// EffectsForOperation returns a page of effects for a given operation id. +func (q *Q) EffectsForOperation(ctx context.Context, id int64, page db2.PageQuery) ([]Effect, error) { start := toid.Parse(id) end := start end.IncOperationOrder() - q.sql = q.sql.Where( + query := selectEffect.Where( "heff.history_operation_id >= ? AND heff.history_operation_id < ?", start.ToInt64(), end.ToInt64(), ) - - return q + return q.selectEffectsPage(ctx, query, page) } -// ForLiquidityPool filters the query to only effects in a specific liquidity pool, -// specified by its id. -func (q *EffectsQ) ForLiquidityPool(ctx context.Context, page db2.PageQuery, id string) *EffectsQ { - if q.Err != nil { - return q - } - +// EffectsForLiquidityPool returns a page of effects for a given liquidity pool. +func (q *Q) EffectsForLiquidityPool(ctx context.Context, id string, page db2.PageQuery) ([]Effect, error) { op, _, err := page.CursorInt64Pair(db2.DefaultPairSep) if err != nil { - q.Err = err - return q + return nil, err } query := `SELECT holp.history_operation_id @@ -150,59 +158,62 @@ func (q *EffectsQ) ForLiquidityPool(ctx context.Context, page db2.PageQuery, id case "desc": query += "AND holp.history_operation_id <= ? ORDER BY holp.history_operation_id desc LIMIT ?" default: - q.Err = errors.Errorf("invalid paging order: %s", page.Order) - return q + return nil, errors.Errorf("invalid paging order: %s", page.Order) } var liquidityPoolOperationIDs []int64 - err = q.parent.SelectRaw(ctx, &liquidityPoolOperationIDs, query, id, op, page.Limit) + err = q.SelectRaw(ctx, &liquidityPoolOperationIDs, query, id, op, page.Limit) if err != nil { - q.Err = err - return q + return nil, err } - q.sql = q.sql.Where(map[string]interface{}{ - "heff.history_operation_id": liquidityPoolOperationIDs, - }) - return q + return q.selectEffectsPage( + ctx, + selectEffect.Where(map[string]interface{}{ + "heff.history_operation_id": liquidityPoolOperationIDs, + }), + page, + ) } -// ForTransaction filters the query to only effects in a specific -// transaction, specified by the transactions's hex-encoded hash. -func (q *EffectsQ) ForTransaction(ctx context.Context, hash string) *EffectsQ { +// EffectsForTransaction returns a page of effects for a given transaction +func (q *Q) EffectsForTransaction(ctx context.Context, hash string, page db2.PageQuery) ([]Effect, error) { var tx Transaction - q.Err = q.parent.TransactionByHash(ctx, &tx, hash) - if q.Err != nil { - return q + if err := q.TransactionByHash(ctx, &tx, hash); err != nil { + return nil, err } start := toid.Parse(tx.ID) end := start end.TransactionOrder++ - q.sql = q.sql.Where( - "heff.history_operation_id >= ? AND heff.history_operation_id < ?", - start.ToInt64(), - end.ToInt64(), - ) - return q + return q.selectEffectsPage( + ctx, + selectEffect.Where("heff.history_operation_id >= ? AND heff.history_operation_id < ?", + start.ToInt64(), + end.ToInt64(), + ), + page, + ) } -// Page specifies the paging constraints for the query being built by `q`. -func (q *EffectsQ) Page(page db2.PageQuery) *EffectsQ { - if q.Err != nil { - return q - } - +func parseEffectsCursor(page db2.PageQuery) (int64, int64, error) { op, idx, err := page.CursorInt64Pair(db2.DefaultPairSep) if err != nil { - q.Err = err - return q + return 0, 0, err } if idx > math.MaxInt32 { idx = math.MaxInt32 } + return op, idx, nil +} + +func (q *Q) selectEffectsPage(ctx context.Context, query sq.SelectBuilder, page db2.PageQuery) ([]Effect, error) { + op, idx, err := parseEffectsCursor(page) + if err != nil { + return nil, err + } // NOTE: Remember to test the queries below with EXPLAIN / EXPLAIN ANALYZE // before changing them. @@ -210,7 +221,7 @@ func (q *EffectsQ) Page(page db2.PageQuery) *EffectsQ { // DB will perform a full table scan. switch page.Order { case "asc": - q.sql = q.sql. + query = query. Where(`( heff.history_operation_id >= ? AND ( @@ -219,7 +230,7 @@ func (q *EffectsQ) Page(page db2.PageQuery) *EffectsQ { ))`, op, op, op, idx). OrderBy("heff.history_operation_id asc, heff.order asc") case "desc": - q.sql = q.sql. + query = query. Where(`( heff.history_operation_id <= ? AND ( @@ -229,18 +240,14 @@ func (q *EffectsQ) Page(page db2.PageQuery) *EffectsQ { OrderBy("heff.history_operation_id desc, heff.order desc") } - q.sql = q.sql.Limit(page.Limit) - return q -} + query = query.Limit(page.Limit) -// Select loads the results of the query specified by `q` into `dest`. -func (q *EffectsQ) Select(ctx context.Context, dest interface{}) error { - if q.Err != nil { - return q.Err + var rows []Effect + if err = q.Select(ctx, &rows, query); err != nil { + return nil, err } - q.Err = q.parent.Select(ctx, dest, q.sql) - return q.Err + return rows, nil } // QEffects defines history_effects related queries. diff --git a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go index dc02148a7d..e1ac998953 100644 --- a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go +++ b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go @@ -6,6 +6,7 @@ import ( "github.com/guregu/null" + "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/toid" ) @@ -42,8 +43,12 @@ func TestAddEffect(t *testing.T) { tt.Assert.NoError(builder.Exec(tt.Ctx, q)) tt.Assert.NoError(q.Commit()) - effects := []Effect{} - tt.Assert.NoError(q.Effects().Select(tt.Ctx, &effects)) + effects, err := q.Effects(tt.Ctx, db2.PageQuery{ + Cursor: "0-0", + Order: "asc", + Limit: 200, + }) + tt.Require.NoError(err) tt.Assert.Len(effects, 1) effect := effects[0] diff --git a/services/horizon/internal/db2/history/effect_test.go b/services/horizon/internal/db2/history/effect_test.go index 498d5e92df..19af0ceff8 100644 --- a/services/horizon/internal/db2/history/effect_test.go +++ b/services/horizon/internal/db2/history/effect_test.go @@ -57,11 +57,11 @@ func TestEffectsForLiquidityPool(t *testing.T) { tt.Assert.NoError(q.Commit()) var result []Effect - err = q.Effects().ForLiquidityPool(tt.Ctx, db2.PageQuery{ + result, err = q.EffectsForLiquidityPool(tt.Ctx, liquidityPoolID, db2.PageQuery{ Cursor: "0-0", Order: "asc", Limit: 10, - }, liquidityPoolID).Select(tt.Ctx, &result) + }) tt.Assert.NoError(err) tt.Assert.Len(result, 1) @@ -156,8 +156,12 @@ func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) { tt.Require.NoError(builder.Exec(tt.Ctx, q)) tt.Assert.NoError(q.Commit()) - var results []Effect - tt.Require.NoError(q.Effects().Select(tt.Ctx, &results)) + results, err := q.Effects(tt.Ctx, db2.PageQuery{ + Cursor: "0-0", + Order: "asc", + Limit: 200, + }) + tt.Require.NoError(err) tt.Require.Len(results, len(tests)) for i, test := range tests { diff --git a/services/horizon/internal/db2/history/main.go b/services/horizon/internal/db2/history/main.go index 41a4cb0068..d9c5ea7557 100644 --- a/services/horizon/internal/db2/history/main.go +++ b/services/horizon/internal/db2/history/main.go @@ -632,14 +632,6 @@ type SequenceBumped struct { NewSeq int64 `json:"new_seq"` } -// EffectsQ is a helper struct to aid in configuring queries that loads -// slices of Ledger structs. -type EffectsQ struct { - Err error - parent *Q - sql sq.SelectBuilder -} - // EffectType is the numeric type for an effect, used as the `type` field in the // `history_effects` table. type EffectType int diff --git a/services/horizon/internal/db2/history/transaction_test.go b/services/horizon/internal/db2/history/transaction_test.go index 4e85624701..65c6734644 100644 --- a/services/horizon/internal/db2/history/transaction_test.go +++ b/services/horizon/internal/db2/history/transaction_test.go @@ -892,12 +892,20 @@ func TestFetchFeeBumpTransaction(t *testing.T) { tt.Assert.Equal(byOuterhash, byInnerHash) } - var outerEffects, innerEffects []Effect - err = q.Effects().ForTransaction(tt.Ctx, fixture.OuterHash).Select(tt.Ctx, &outerEffects) + var innerEffects []Effect + outerEffects, err := q.EffectsForTransaction(tt.Ctx, fixture.OuterHash, db2.PageQuery{ + Cursor: "0-0", + Order: "asc", + Limit: 200, + }) tt.Assert.NoError(err) tt.Assert.Len(outerEffects, 1) - err = q.Effects().ForTransaction(tt.Ctx, fixture.InnerHash).Select(tt.Ctx, &innerEffects) + innerEffects, err = q.EffectsForTransaction(tt.Ctx, fixture.InnerHash, db2.PageQuery{ + Cursor: "0-0", + Order: "asc", + Limit: 200, + }) tt.Assert.NoError(err) tt.Assert.Equal(outerEffects, innerEffects) } From d3086517567cf6b8f64a6116ba56d881c943f87c Mon Sep 17 00:00:00 2001 From: tamirms Date: Mon, 11 Mar 2024 17:05:15 +0000 Subject: [PATCH 05/11] speed up asset loader query (#5237) --- .../horizon/internal/db2/history/asset_loader.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/services/horizon/internal/db2/history/asset_loader.go b/services/horizon/internal/db2/history/asset_loader.go index bdf40fb843..fe17dc17be 100644 --- a/services/horizon/internal/db2/history/asset_loader.go +++ b/services/horizon/internal/db2/history/asset_loader.go @@ -7,8 +7,6 @@ import ( "sort" "strings" - sq "github.com/Masterminds/squirrel" - "github.com/stellar/go/support/collections/set" "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" @@ -108,13 +106,17 @@ func (a *AssetLoader) lookupKeys(ctx context.Context, q *Q, keys []AssetKey) err for i := 0; i < len(keys); i += loaderLookupBatchSize { end := ordered.Min(len(keys), i+loaderLookupBatchSize) subset := keys[i:end] - keyStrings := make([]string, 0, len(subset)) + args := make([]interface{}, 0, 3*len(subset)) + placeHolders := make([]string, 0, len(subset)) for _, key := range subset { - keyStrings = append(keyStrings, key.Type+"/"+key.Code+"/"+key.Issuer) + args = append(args, key.Code, key.Type, key.Issuer) + placeHolders = append(placeHolders, "(?, ?, ?)") } - err := q.Select(ctx, &rows, sq.Select("*").From("history_assets").Where(sq.Eq{ - "concat(asset_type, '/', asset_code, '/', asset_issuer)": keyStrings, - })) + rawSQL := fmt.Sprintf( + "SELECT * FROM history_assets WHERE (asset_code, asset_type, asset_issuer) in (%s)", + strings.Join(placeHolders, ", "), + ) + err := q.SelectRaw(ctx, &rows, rawSQL, args...) if err != nil { return errors.Wrap(err, "could not select assets") } From 4b0b0787e00851a0da6769c20f5e66db0d848400 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Mon, 11 Mar 2024 13:14:32 -0700 Subject: [PATCH 06/11] updated CHANGELOG for 2.29.0 --- services/horizon/CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 4003073503..50c1407d26 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -5,12 +5,21 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +## 2.29.0 + ### Added - New `db_error_total` metrics key with labels `ctx_error`, `db_error`, and `db_error_extra` ([5225](https://github.com/stellar/go/pull/5225)). -- Bumped go version to the latest (1.22.1) +- Bumped go version to the latest (1.22.1) ([5232](https://github.com/stellar/go/pull/5232)) +- Add metrics for ingestion loaders ([5209](https://github.com/stellar/go/pull/5209)). +- Add metrics for http api requests in flight and requests received ([5240](https://github.com/stellar/go/pull/5240)). ### Fixed - History archive access is more effective when you pass list of URLs to Horizon: they will now be accessed in a round-robin fashion, use alternative archives on errors, and intelligently back off ([5224](https://github.com/stellar/go/pull/5224)) +- Remove captive core info request error logs ([5145](https://github.com/stellar/go/pull/5145)) +- Removed duplicate "Processed Ledger" log statement during resume state ([5152](https://github.com/stellar/go/pull/5152)) +- Fixed incorrect duration for ingestion processor metric ([5216](https://github.com/stellar/go/pull/5216)) +- Fixed sql performance on account transactions query ([5229](https://github.com/stellar/go/pull/5229)) + ### Breaking Changes - The Horizon API Transaction resource field in json `result_meta_xdr` is now optional and Horizon API will not emit the field when Horizon has been configured with `SKIP_TXMETA=true`, effectively null, otherwise if Horizon is configured with `SKIP_TXMETA=false` which is default, then the API Transaction field `result_meta_xdr` will remain present and populated with base64 encoded xdr [5228](https://github.com/stellar/go/pull/5228). From 71fef3cca734362a8e1c2658a169d944a4503157 Mon Sep 17 00:00:00 2001 From: tamirms Date: Thu, 14 Mar 2024 09:52:05 +0000 Subject: [PATCH 07/11] services/horizon/internal/ingest/processors: Fix bug in claimable balance change processor (#5246) Co-authored-by: Urvi --- .../db2/history/claimable_balances.go | 61 ++++++++++++++ .../db2/history/claimable_balances_test.go | 79 +++++++++++++++++++ .../db2/history/mock_q_claimable_balances.go | 5 ++ .../claimable_balances_change_processor.go | 19 ++++- ...laimable_balances_change_processor_test.go | 71 ++++++++++++++++- 5 files changed, 230 insertions(+), 5 deletions(-) diff --git a/services/horizon/internal/db2/history/claimable_balances.go b/services/horizon/internal/db2/history/claimable_balances.go index c198ee162d..abdf4ed758 100644 --- a/services/horizon/internal/db2/history/claimable_balances.go +++ b/services/horizon/internal/db2/history/claimable_balances.go @@ -140,6 +140,7 @@ type Claimant struct { // QClaimableBalances defines claimable-balance-related related queries. type QClaimableBalances interface { + UpsertClaimableBalances(ctx context.Context, cb []ClaimableBalance) error RemoveClaimableBalances(ctx context.Context, ids []string) (int64, error) RemoveClaimableBalanceClaimants(ctx context.Context, ids []string) (int64, error) GetClaimableBalancesByID(ctx context.Context, ids []string) ([]ClaimableBalance, error) @@ -185,6 +186,66 @@ func (q *Q) GetClaimantsByClaimableBalances(ctx context.Context, ids []string) ( return claimantsMap, err } +// UpsertClaimableBalances upserts a batch of claimable balances in the claimable_balances table. +// It also upserts the corresponding claimants in the claimable_balance_claimants table. +func (q *Q) UpsertClaimableBalances(ctx context.Context, cbs []ClaimableBalance) error { + if err := q.upsertCBs(ctx, cbs); err != nil { + return errors.Wrap(err, "could not upsert claimable balances") + } + + if err := q.upsertCBClaimants(ctx, cbs); err != nil { + return errors.Wrap(err, "could not upsert claimable balance claimants") + } + + return nil +} + +func (q *Q) upsertCBClaimants(ctx context.Context, cbs []ClaimableBalance) error { + var id, lastModifiedLedger, destination []interface{} + + for _, cb := range cbs { + for _, claimant := range cb.Claimants { + id = append(id, cb.BalanceID) + lastModifiedLedger = append(lastModifiedLedger, cb.LastModifiedLedger) + destination = append(destination, claimant.Destination) + } + } + + upsertFields := []upsertField{ + {"id", "text", id}, + {"destination", "text", destination}, + {"last_modified_ledger", "integer", lastModifiedLedger}, + } + + return q.upsertRows(ctx, "claimable_balance_claimants", "id, destination", upsertFields) +} + +func (q *Q) upsertCBs(ctx context.Context, cbs []ClaimableBalance) error { + var id, claimants, asset, amount, sponsor, lastModifiedLedger, flags []interface{} + + for _, cb := range cbs { + id = append(id, cb.BalanceID) + claimants = append(claimants, cb.Claimants) + asset = append(asset, cb.Asset) + amount = append(amount, cb.Amount) + sponsor = append(sponsor, cb.Sponsor) + lastModifiedLedger = append(lastModifiedLedger, cb.LastModifiedLedger) + flags = append(flags, cb.Flags) + } + + upsertFields := []upsertField{ + {"id", "text", id}, + {"claimants", "jsonb", claimants}, + {"asset", "text", asset}, + {"amount", "bigint", amount}, + {"sponsor", "text", sponsor}, + {"last_modified_ledger", "integer", lastModifiedLedger}, + {"flags", "int", flags}, + } + + return q.upsertRows(ctx, "claimable_balances", "id", upsertFields) +} + // RemoveClaimableBalances deletes claimable balances table. // Returns number of rows affected and error. func (q *Q) RemoveClaimableBalances(ctx context.Context, ids []string) (int64, error) { diff --git a/services/horizon/internal/db2/history/claimable_balances_test.go b/services/horizon/internal/db2/history/claimable_balances_test.go index 2e6d621945..1ffe442244 100644 --- a/services/horizon/internal/db2/history/claimable_balances_test.go +++ b/services/horizon/internal/db2/history/claimable_balances_test.go @@ -588,6 +588,85 @@ func TestFindClaimableBalancesByDestinationWithLimit(t *testing.T) { }) } +func TestUpdateClaimableBalance(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + q := &Q{tt.HorizonSession()} + + accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" + lastModifiedLedgerSeq := xdr.Uint32(123) + asset := xdr.MustNewCreditAsset("USD", accountID) + balanceID := xdr.ClaimableBalanceId{ + Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0, + V0: &xdr.Hash{1, 2, 3}, + } + id, err := xdr.MarshalHex(balanceID) + tt.Assert.NoError(err) + cBalance := ClaimableBalance{ + BalanceID: id, + Claimants: []Claimant{ + { + Destination: accountID, + Predicate: xdr.ClaimPredicate{ + Type: xdr.ClaimPredicateTypeClaimPredicateUnconditional, + }, + }, + }, + Asset: asset, + LastModifiedLedger: 123, + Amount: 10, + } + + err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance}) + tt.Assert.NoError(err) + + cBalancesClaimants, err := q.GetClaimantsByClaimableBalances(tt.Ctx, []string{cBalance.BalanceID}) + tt.Assert.NoError(err) + tt.Assert.Len(cBalancesClaimants[cBalance.BalanceID], 1) + tt.Assert.Equal(ClaimableBalanceClaimant{ + BalanceID: cBalance.BalanceID, + Destination: accountID, + LastModifiedLedger: cBalance.LastModifiedLedger, + }, cBalancesClaimants[cBalance.BalanceID][0]) + + // add sponsor + cBalance2 := ClaimableBalance{ + BalanceID: id, + Claimants: []Claimant{ + { + Destination: accountID, + Predicate: xdr.ClaimPredicate{ + Type: xdr.ClaimPredicateTypeClaimPredicateUnconditional, + }, + }, + }, + Asset: asset, + LastModifiedLedger: 123 + 1, + Amount: 10, + Sponsor: null.StringFrom("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), + } + + err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance2}) + tt.Assert.NoError(err) + + cbs := []ClaimableBalance{} + err = q.Select(tt.Ctx, &cbs, selectClaimableBalances) + tt.Assert.NoError(err) + tt.Assert.Len(cbs, 1) + tt.Assert.Equal("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML", cbs[0].Sponsor.String) + tt.Assert.Equal(uint32(lastModifiedLedgerSeq+1), cbs[0].LastModifiedLedger) + + cBalancesClaimants, err = q.GetClaimantsByClaimableBalances(tt.Ctx, []string{cBalance2.BalanceID}) + tt.Assert.NoError(err) + tt.Assert.Len(cBalancesClaimants[cBalance2.BalanceID], 1) + tt.Assert.Equal(ClaimableBalanceClaimant{ + BalanceID: cBalance2.BalanceID, + Destination: accountID, + LastModifiedLedger: cBalance2.LastModifiedLedger, + }, cBalancesClaimants[cBalance2.BalanceID][0]) +} + func TestFindClaimableBalance(t *testing.T) { tt := test.Start(t) defer tt.Finish() diff --git a/services/horizon/internal/db2/history/mock_q_claimable_balances.go b/services/horizon/internal/db2/history/mock_q_claimable_balances.go index 64b65cf1a3..6a3adffac1 100644 --- a/services/horizon/internal/db2/history/mock_q_claimable_balances.go +++ b/services/horizon/internal/db2/history/mock_q_claimable_balances.go @@ -21,6 +21,11 @@ func (m *MockQClaimableBalances) GetClaimableBalancesByID(ctx context.Context, i return a.Get(0).([]ClaimableBalance), a.Error(1) } +func (m *MockQClaimableBalances) UpsertClaimableBalances(ctx context.Context, cbs []ClaimableBalance) error { + a := m.Called(ctx, cbs) + return a.Error(0) +} + func (m *MockQClaimableBalances) RemoveClaimableBalances(ctx context.Context, ids []string) (int64, error) { a := m.Called(ctx, ids) return a.Get(0).(int64), a.Error(1) diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go index de729f9605..fce002881c 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go @@ -2,7 +2,6 @@ package processors import ( "context" - "fmt" "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" @@ -60,7 +59,8 @@ func (p *ClaimableBalancesChangeProcessor) ProcessChange(ctx context.Context, ch func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { defer p.reset() var ( - cbIDsToDelete []string + cbIDsToDelete []string + updatedBalances []history.ClaimableBalance ) changes := p.cache.GetChanges() for _, change := range changes { @@ -97,8 +97,13 @@ func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { } cbIDsToDelete = append(cbIDsToDelete, id) default: - // claimable balance can only be created or removed - return fmt.Errorf("invalid change entry for a claimable balance was detected") + // this case should only occur if the sponsor has changed in the claimable balance + // the other fields of a claimable balance are immutable + postCB, err := p.ledgerEntryToRow(change.Post) + if err != nil { + return err + } + updatedBalances = append(updatedBalances, postCB) } } @@ -112,6 +117,12 @@ func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { return errors.Wrap(err, "error executing ClaimableBalanceBatchInsertBuilder") } + if len(updatedBalances) > 0 { + if err = p.qClaimableBalances.UpsertClaimableBalances(ctx, updatedBalances); err != nil { + return errors.Wrap(err, "error updating claimable balances") + } + } + if len(cbIDsToDelete) > 0 { count, err := p.qClaimableBalances.RemoveClaimableBalances(ctx, cbIDsToDelete) if err != nil { diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go index 524de095f7..a10cc9db7d 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go @@ -8,10 +8,11 @@ import ( "github.com/guregu/null" + "github.com/stretchr/testify/suite" + "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/xdr" - "github.com/stretchr/testify/suite" ) func TestClaimableBalancesChangeProcessorTestSuiteState(t *testing.T) { @@ -249,3 +250,71 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TestRemoveClaimableBal []string{id}, ).Return(int64(1), nil).Once() } + +func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TestUpdateClaimableBalanceAddSponsor() { + balanceID := xdr.ClaimableBalanceId{ + Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0, + V0: &xdr.Hash{1, 2, 3}, + } + cBalance := xdr.ClaimableBalanceEntry{ + BalanceId: balanceID, + Claimants: []xdr.Claimant{}, + Asset: xdr.MustNewCreditAsset("USD", "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), + Amount: 10, + } + lastModifiedLedgerSeq := xdr.Uint32(123) + + pre := xdr.LedgerEntry{ + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeClaimableBalance, + ClaimableBalance: &cBalance, + }, + LastModifiedLedgerSeq: lastModifiedLedgerSeq - 1, + Ext: xdr.LedgerEntryExt{ + V: 1, + V1: &xdr.LedgerEntryExtensionV1{ + SponsoringId: nil, + }, + }, + } + + // add sponsor + updated := xdr.LedgerEntry{ + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeClaimableBalance, + ClaimableBalance: &cBalance, + }, + LastModifiedLedgerSeq: lastModifiedLedgerSeq, + Ext: xdr.LedgerEntryExt{ + V: 1, + V1: &xdr.LedgerEntryExtensionV1{ + SponsoringId: xdr.MustAddressPtr("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), + }, + }, + } + s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() + + err := s.processor.ProcessChange(s.ctx, ingest.Change{ + Type: xdr.LedgerEntryTypeClaimableBalance, + Pre: &pre, + Post: &updated, + }) + s.Assert().NoError(err) + + id, err := xdr.MarshalHex(balanceID) + s.Assert().NoError(err) + s.mockQ.On( + "UpsertClaimableBalances", + s.ctx, + []history.ClaimableBalance{ + { + BalanceID: id, + Claimants: []history.Claimant{}, + Asset: cBalance.Asset, + Amount: cBalance.Amount, + LastModifiedLedger: uint32(lastModifiedLedgerSeq), + Sponsor: null.StringFrom("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), + }, + }, + ).Return(nil).Once() +} From 50677eeedca9d3e968178546c874a11cc2c302a1 Mon Sep 17 00:00:00 2001 From: tamirms Date: Thu, 14 Mar 2024 10:21:04 +0000 Subject: [PATCH 08/11] support/db: Delay canceling queries from client side when there's a statement / transaction timeout configured in postgres (#5223) --- services/horizon/internal/app.go | 1 + services/horizon/internal/config.go | 1 + .../internal/db2/history/liquidity_pools.go | 5 +- .../db2/history/liquidity_pools_test.go | 1 + .../horizon/internal/db2/history/offers.go | 4 +- services/horizon/internal/flags.go | 31 +++++ services/horizon/internal/flags_test.go | 96 +++++++++++++--- services/horizon/internal/httpx/middleware.go | 13 ++- services/horizon/internal/httpx/router.go | 6 +- services/horizon/internal/init.go | 22 ++-- services/horizon/internal/middleware_test.go | 2 +- staticcheck.sh | 2 +- support/config/config_option.go | 4 +- support/config/config_option_test.go | 18 +-- support/db/main.go | 6 +- support/db/mock_session.go | 8 +- support/db/session.go | 106 ++++++++++++++++-- support/db/session_test.go | 43 ++++++- 18 files changed, 297 insertions(+), 72 deletions(-) diff --git a/services/horizon/internal/app.go b/services/horizon/internal/app.go index 59a8fb4432..ed338f2ac3 100644 --- a/services/horizon/internal/app.go +++ b/services/horizon/internal/app.go @@ -532,6 +532,7 @@ func (a *App) init() error { SSEUpdateFrequency: a.config.SSEUpdateFrequency, StaleThreshold: a.config.StaleThreshold, ConnectionTimeout: a.config.ConnectionTimeout, + ClientQueryTimeout: a.config.ClientQueryTimeout, MaxHTTPRequestSize: a.config.MaxHTTPRequestSize, NetworkPassphrase: a.config.NetworkPassphrase, MaxPathLength: a.config.MaxPathLength, diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go index f1f8ac078d..eb89d72efd 100644 --- a/services/horizon/internal/config.go +++ b/services/horizon/internal/config.go @@ -38,6 +38,7 @@ type Config struct { SSEUpdateFrequency time.Duration ConnectionTimeout time.Duration + ClientQueryTimeout time.Duration // MaxHTTPRequestSize is the maximum allowed request payload size MaxHTTPRequestSize uint RateQuota *throttled.RateQuota diff --git a/services/horizon/internal/db2/history/liquidity_pools.go b/services/horizon/internal/db2/history/liquidity_pools.go index 46e6ba59d3..3259163a89 100644 --- a/services/horizon/internal/db2/history/liquidity_pools.go +++ b/services/horizon/internal/db2/history/liquidity_pools.go @@ -9,8 +9,9 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/guregu/null" - "github.com/jmoiron/sqlx" + "github.com/stellar/go/services/horizon/internal/db2" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) @@ -188,7 +189,7 @@ func (q *Q) GetLiquidityPools(ctx context.Context, query LiquidityPoolsQuery) ([ } func (q *Q) StreamAllLiquidityPools(ctx context.Context, callback func(LiquidityPool) error) error { - var rows *sqlx.Rows + var rows *db.Rows var err error if rows, err = q.Query(ctx, selectLiquidityPools.Where("deleted = ?", false)); err != nil { diff --git a/services/horizon/internal/db2/history/liquidity_pools_test.go b/services/horizon/internal/db2/history/liquidity_pools_test.go index fd268d2518..2488945168 100644 --- a/services/horizon/internal/db2/history/liquidity_pools_test.go +++ b/services/horizon/internal/db2/history/liquidity_pools_test.go @@ -112,6 +112,7 @@ func TestStreamAllLiquidity(t *testing.T) { pools = append(pools, pool) return nil }) + tt.Assert.NoError(err) sort.Slice(pools, func(i, j int) bool { return pools[i].PoolID < pools[j].PoolID }) diff --git a/services/horizon/internal/db2/history/offers.go b/services/horizon/internal/db2/history/offers.go index 98a08fef87..1dcfe25321 100644 --- a/services/horizon/internal/db2/history/offers.go +++ b/services/horizon/internal/db2/history/offers.go @@ -5,8 +5,8 @@ import ( "database/sql" sq "github.com/Masterminds/squirrel" - "github.com/jmoiron/sqlx" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" ) @@ -108,7 +108,7 @@ func (q *Q) StreamAllOffers(ctx context.Context, callback func(Offer) error) err } func (q *Q) streamAllOffersBatch(ctx context.Context, lastId int64, limit uint64, callback func(Offer) error) (int64, error) { - var rows *sqlx.Rows + var rows *db.Rows var err error rows, err = q.Query(ctx, selectOffers. diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 4c8e4dc2f5..48a83057f4 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "strings" + "time" "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -69,6 +70,7 @@ const ( StellarTestnet = "testnet" defaultMaxHTTPRequestSize = uint(200 * 1024) + clientQueryTimeoutNotSet = -1 ) var ( @@ -437,6 +439,30 @@ func Flags() (*Config, support.ConfigOptions) { Usage: "defines the timeout of connection after which 504 response will be sent or stream will be closed, if Horizon is behind a load balancer with idle connection timeout, this should be set to a few seconds less that idle timeout, does not apply to POST /transactions", UsedInCommands: ApiServerCommands, }, + &support.ConfigOption{ + Name: "client-query-timeout", + ConfigKey: &config.ClientQueryTimeout, + OptType: types.Int, + FlagDefault: clientQueryTimeoutNotSet, + CustomSetValue: func(co *support.ConfigOption) error { + if !support.IsExplicitlySet(co) { + *(co.ConfigKey.(*time.Duration)) = time.Duration(co.FlagDefault.(int)) + return nil + } + duration := viper.GetInt(co.Name) + if duration < 0 { + return fmt.Errorf("%s cannot be negative", co.Name) + } + *(co.ConfigKey.(*time.Duration)) = time.Duration(duration) * time.Second + return nil + }, + Usage: "defines the timeout for when horizon will cancel all postgres queries connected to an HTTP request. The timeout is measured in seconds since the start of the HTTP request. Note, this timeout does not apply to POST /transactions. " + + "The difference between client-query-timeout and connection-timeout is that connection-timeout applies a postgres statement timeout whereas client-query-timeout will send an additional request to postgres to cancel the ongoing query. " + + "Generally, client-query-timeout should be configured to be higher than connection-timeout to allow the postgres statement timeout to kill long running queries without having to send the additional cancel request to postgres. " + + "By default, client-query-timeout will be set to twice the connection-timeout. Setting client-query-timeout to 0 will disable the timeout which means that Horizon will never kill long running queries using the cancel request, however, " + + "long running queries can still be killed through the postgres statement timeout which is configured via the connection-timeout flag.", + UsedInCommands: ApiServerCommands, + }, &support.ConfigOption{ Name: "max-http-request-size", ConfigKey: &config.MaxHTTPRequestSize, @@ -983,5 +1009,10 @@ func ApplyFlags(config *Config, flags support.ConfigOptions, options ApplyOption " If Horizon is behind both, use --behind-cloudflare only") } + if config.ClientQueryTimeout == clientQueryTimeoutNotSet { + // the default value for cancel-db-query-timeout is twice the connection-timeout + config.ClientQueryTimeout = config.ConnectionTimeout * 2 + } + return nil } diff --git a/services/horizon/internal/flags_test.go b/services/horizon/internal/flags_test.go index ef2d5d3a02..3da39bc7a5 100644 --- a/services/horizon/internal/flags_test.go +++ b/services/horizon/internal/flags_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "testing" + "time" "github.com/spf13/cobra" @@ -207,22 +208,69 @@ func Test_createCaptiveCoreConfig(t *testing.T) { } } -func TestEnvironmentVariables(t *testing.T) { - environmentVars := map[string]string{ - "INGEST": "false", - "HISTORY_ARCHIVE_URLS": "http://localhost:1570", - "DATABASE_URL": "postgres://postgres@localhost/test_332cb65e6b00?sslmode=disable&timezone=UTC", - "STELLAR_CORE_URL": "http://localhost:11626", - "NETWORK_PASSPHRASE": "Standalone Network ; February 2017", - "APPLY_MIGRATIONS": "true", - "CHECKPOINT_FREQUENCY": "8", - "MAX_DB_CONNECTIONS": "50", - "ADMIN_PORT": "6060", - "PORT": "8001", - "CAPTIVE_CORE_BINARY_PATH": os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN"), - "CAPTIVE_CORE_CONFIG_PATH": "../docker/captive-core-classic-integration-tests.cfg", - "CAPTIVE_CORE_USE_DB": "true", +func TestClientQueryTimeoutFlag(t *testing.T) { + for _, testCase := range []struct { + name string + flag string + parsed time.Duration + err string + }{ + { + "negative value", + "-1", + 0, + "client-query-timeout cannot be negative", + }, + { + "default value", + "", + time.Second * 110, + "", + }, + { + "custom value", + "20", + time.Second * 20, + "", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + environmentVars := horizonEnvVars() + if testCase.flag != "" { + environmentVars["CLIENT_QUERY_TIMEOUT"] = testCase.flag + } + + envManager := test.NewEnvironmentManager() + defer func() { + envManager.Restore() + }() + if err := envManager.InitializeEnvironmentVariables(environmentVars); err != nil { + require.NoError(t, err) + } + + config, flags := Flags() + horizonCmd := &cobra.Command{ + Use: "horizon", + Short: "Client-facing api server for the Stellar network", + SilenceErrors: true, + SilenceUsage: true, + Long: "Client-facing API server for the Stellar network.", + } + if err := flags.Init(horizonCmd); err != nil { + require.NoError(t, err) + } + if err := ApplyFlags(config, flags, ApplyOptions{RequireCaptiveCoreFullConfig: true, AlwaysIngest: false}); err != nil { + require.EqualError(t, err, testCase.err) + } else { + require.Empty(t, testCase.err) + } + require.Equal(t, testCase.parsed, config.ClientQueryTimeout) + }) } +} + +func TestEnvironmentVariables(t *testing.T) { + environmentVars := horizonEnvVars() envManager := test.NewEnvironmentManager() defer func() { @@ -261,6 +309,24 @@ func TestEnvironmentVariables(t *testing.T) { assert.Equal(t, config.CaptiveCoreConfigUseDB, true) } +func horizonEnvVars() map[string]string { + return map[string]string{ + "INGEST": "false", + "HISTORY_ARCHIVE_URLS": "http://localhost:1570", + "DATABASE_URL": "postgres://postgres@localhost/test_332cb65e6b00?sslmode=disable&timezone=UTC", + "STELLAR_CORE_URL": "http://localhost:11626", + "NETWORK_PASSPHRASE": "Standalone Network ; February 2017", + "APPLY_MIGRATIONS": "true", + "CHECKPOINT_FREQUENCY": "8", + "MAX_DB_CONNECTIONS": "50", + "ADMIN_PORT": "6060", + "PORT": "8001", + "CAPTIVE_CORE_BINARY_PATH": os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN"), + "CAPTIVE_CORE_CONFIG_PATH": "../docker/captive-core-classic-integration-tests.cfg", + "CAPTIVE_CORE_USE_DB": "true", + } +} + func TestRemovedFlags(t *testing.T) { tests := []struct { name string diff --git a/services/horizon/internal/httpx/middleware.go b/services/horizon/internal/httpx/middleware.go index eaa49634a5..9240ed0833 100644 --- a/services/horizon/internal/httpx/middleware.go +++ b/services/horizon/internal/httpx/middleware.go @@ -197,7 +197,7 @@ func recoverMiddleware(h http.Handler) http.Handler { // NewHistoryMiddleware adds session to the request context and ensures Horizon // is not in a stale state, which is when the difference between latest core // ledger and latest history ledger is higher than the given threshold -func NewHistoryMiddleware(ledgerState *ledger.State, staleThreshold int32, session db.SessionInterface) func(http.Handler) http.Handler { +func NewHistoryMiddleware(ledgerState *ledger.State, staleThreshold int32, session db.SessionInterface, contextDBTimeout time.Duration) func(http.Handler) http.Handler { return func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -205,6 +205,7 @@ func NewHistoryMiddleware(ledgerState *ledger.State, staleThreshold int32, sessi if routePattern := supportHttp.GetChiRoutePattern(r); routePattern != "" { ctx = context.WithValue(ctx, &db.RouteContextKey, routePattern) } + ctx = setContextDBTimeout(contextDBTimeout, ctx) if staleThreshold > 0 { ls := ledgerState.CurrentStatus() isStale := (ls.CoreLatest - ls.HistoryLatest) > int32(staleThreshold) @@ -238,6 +239,7 @@ func NewHistoryMiddleware(ledgerState *ledger.State, staleThreshold int32, sessi // returning invalid data to the user) type StateMiddleware struct { HorizonSession db.SessionInterface + ClientQueryTimeout time.Duration NoStateVerification bool } @@ -276,6 +278,7 @@ func (m *StateMiddleware) WrapFunc(h http.HandlerFunc) http.HandlerFunc { if routePattern := supportHttp.GetChiRoutePattern(r); routePattern != "" { ctx = context.WithValue(ctx, &db.RouteContextKey, routePattern) } + ctx = setContextDBTimeout(m.ClientQueryTimeout, ctx) session := m.HorizonSession.Clone() q := &history.Q{session} sseRequest := render.Negotiate(r) == render.MimeEventStream @@ -344,6 +347,14 @@ func (m *StateMiddleware) WrapFunc(h http.HandlerFunc) http.HandlerFunc { } } +func setContextDBTimeout(timeout time.Duration, ctx context.Context) context.Context { + var deadline time.Time + if timeout > 0 { + deadline = time.Now().Add(timeout) + } + return context.WithValue(ctx, &db.DeadlineCtxKey, deadline) +} + // WrapFunc executes the middleware on a given HTTP handler function func (m *StateMiddleware) Wrap(h http.Handler) http.Handler { return m.WrapFunc(h.ServeHTTP) diff --git a/services/horizon/internal/httpx/router.go b/services/horizon/internal/httpx/router.go index 2755b9e062..60cf617468 100644 --- a/services/horizon/internal/httpx/router.go +++ b/services/horizon/internal/httpx/router.go @@ -38,6 +38,7 @@ type RouterConfig struct { SSEUpdateFrequency time.Duration StaleThreshold uint ConnectionTimeout time.Duration + ClientQueryTimeout time.Duration MaxHTTPRequestSize uint NetworkPassphrase string MaxPathLength uint @@ -139,7 +140,8 @@ func (r *Router) addMiddleware(config *RouterConfig, func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRateLimiter, ledgerState *ledger.State) { stateMiddleware := StateMiddleware{ - HorizonSession: config.DBSession, + HorizonSession: config.DBSession, + ClientQueryTimeout: config.ClientQueryTimeout, } r.Method(http.MethodGet, "/health", config.HealthCheck) @@ -157,7 +159,7 @@ func (r *Router) addRoutes(config *RouterConfig, rateLimiter *throttled.HTTPRate LedgerSourceFactory: historyLedgerSourceFactory{ledgerState: ledgerState, updateFrequency: config.SSEUpdateFrequency}, } - historyMiddleware := NewHistoryMiddleware(ledgerState, int32(config.StaleThreshold), config.DBSession) + historyMiddleware := NewHistoryMiddleware(ledgerState, int32(config.StaleThreshold), config.DBSession, config.ClientQueryTimeout) // State endpoints behind stateMiddleware r.Group(func(r chi.Router) { r.Route("/accounts", func(r chi.Router) { diff --git a/services/horizon/internal/init.go b/services/horizon/internal/init.go index 0c0fe2c2cd..2311833508 100644 --- a/services/horizon/internal/init.go +++ b/services/horizon/internal/init.go @@ -45,38 +45,29 @@ func mustInitHorizonDB(app *App) { log.Fatalf("max open connections to horizon db must be greater than %d", ingest.MaxDBConnections) } } + serverSidePGTimeoutConfigs := []db.ClientConfig{ + db.StatementTimeout(app.config.ConnectionTimeout), + db.IdleTransactionTimeout(app.config.ConnectionTimeout), + } if app.config.RoDatabaseURL == "" { - var clientConfigs []db.ClientConfig - if !app.config.Ingest { - // if we are not ingesting then we don't expect to have long db queries / transactions - clientConfigs = append( - clientConfigs, - db.StatementTimeout(app.config.ConnectionTimeout), - db.IdleTransactionTimeout(app.config.ConnectionTimeout), - ) - } app.historyQ = &history.Q{mustNewDBSession( db.HistorySubservice, app.config.DatabaseURL, maxIdle, maxOpen, app.prometheusRegistry, - clientConfigs..., + serverSidePGTimeoutConfigs..., )} } else { // If RO set, use it for all DB queries - roClientConfigs := []db.ClientConfig{ - db.StatementTimeout(app.config.ConnectionTimeout), - db.IdleTransactionTimeout(app.config.ConnectionTimeout), - } app.historyQ = &history.Q{mustNewDBSession( db.HistorySubservice, app.config.RoDatabaseURL, maxIdle, maxOpen, app.prometheusRegistry, - roClientConfigs..., + serverSidePGTimeoutConfigs..., )} app.primaryHistoryQ = &history.Q{mustNewDBSession( @@ -85,6 +76,7 @@ func mustInitHorizonDB(app *App) { maxIdle, maxOpen, app.prometheusRegistry, + serverSidePGTimeoutConfigs..., )} } } diff --git a/services/horizon/internal/middleware_test.go b/services/horizon/internal/middleware_test.go index 40a74ea69e..269269bf8e 100644 --- a/services/horizon/internal/middleware_test.go +++ b/services/horizon/internal/middleware_test.go @@ -402,7 +402,7 @@ func TestCheckHistoryStaleMiddleware(t *testing.T) { } ledgerState := &ledger.State{} ledgerState.SetStatus(state) - historyMiddleware := httpx.NewHistoryMiddleware(ledgerState, testCase.staleThreshold, tt.HorizonSession()) + historyMiddleware := httpx.NewHistoryMiddleware(ledgerState, testCase.staleThreshold, tt.HorizonSession(), 0) handler := chi.NewRouter() handler.With(historyMiddleware).MethodFunc("GET", "/", endpoint) w := httptest.NewRecorder() diff --git a/staticcheck.sh b/staticcheck.sh index 539641a4b3..5e07d0d026 100755 --- a/staticcheck.sh +++ b/staticcheck.sh @@ -1,7 +1,7 @@ #! /bin/bash set -e -version='2023.1.1' +version='2023.1.7' staticcheck='go run honnef.co/go/tools/cmd/staticcheck@'"$version" diff --git a/support/config/config_option.go b/support/config/config_option.go index 97df9ef956..1fab1910d7 100644 --- a/support/config/config_option.go +++ b/support/config/config_option.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" + "github.com/stellar/go/support/errors" "github.com/stellar/go/support/strutils" ) @@ -248,13 +249,12 @@ func parseEnvVars(entries []string) map[string]bool { return set } -var envVars = parseEnvVars(os.Environ()) - // IsExplicitlySet returns true if and only if the given config option was set explicitly either // via a command line argument or via an environment variable func IsExplicitlySet(co *ConfigOption) bool { // co.flag.Changed is only set to true when the configuration is set via command line parameter. // In the case where a variable is configured via environment variable we need to check envVars. + envVars := parseEnvVars(os.Environ()) return co.flag.Changed || envVars[co.EnvVar] } diff --git a/support/config/config_option_test.go b/support/config/config_option_test.go index 30be01424b..418a50267e 100644 --- a/support/config/config_option_test.go +++ b/support/config/config_option_test.go @@ -88,13 +88,8 @@ func TestConfigOption_optionalFlags_env_set_empty(t *testing.T) { } configOpts.Init(cmd) - prev := envVars - envVars = map[string]bool{ - "STRING": true, - } - defer func() { - envVars = prev - }() + defer os.Setenv("STRING", os.Getenv("STRING")) + os.Setenv("STRING", "") cmd.Execute() assert.Equal(t, "", *optString) @@ -118,15 +113,6 @@ func TestConfigOption_optionalFlags_env_set(t *testing.T) { } configOpts.Init(cmd) - prev := envVars - envVars = map[string]bool{ - "STRING": true, - "UINT": true, - } - defer func() { - envVars = prev - }() - defer os.Setenv("STRING", os.Getenv("STRING")) defer os.Setenv("UINT", os.Getenv("UINT")) os.Setenv("STRING", "str") diff --git a/support/db/main.go b/support/db/main.go index dca23526ee..4b0b4c8b84 100644 --- a/support/db/main.go +++ b/support/db/main.go @@ -21,6 +21,7 @@ import ( "github.com/Masterminds/squirrel" "github.com/jmoiron/sqlx" + "github.com/stellar/go/support/errors" // Enable postgres @@ -119,6 +120,7 @@ type Session struct { DB *sqlx.DB tx *sqlx.Tx + txCancel context.CancelFunc txOptions *sql.TxOptions errorHandlers []ErrorHandlerFunc } @@ -140,8 +142,8 @@ type SessionInterface interface { GetRaw(ctx context.Context, dest interface{}, query string, args ...interface{}) error Select(ctx context.Context, dest interface{}, query squirrel.Sqlizer) error SelectRaw(ctx context.Context, dest interface{}, query string, args ...interface{}) error - Query(ctx context.Context, query squirrel.Sqlizer) (*sqlx.Rows, error) - QueryRaw(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) + Query(ctx context.Context, query squirrel.Sqlizer) (*Rows, error) + QueryRaw(ctx context.Context, query string, args ...interface{}) (*Rows, error) GetTable(name string) *Table Exec(ctx context.Context, query squirrel.Sqlizer) (sql.Result, error) ExecRaw(ctx context.Context, query string, args ...interface{}) (sql.Result, error) diff --git a/support/db/mock_session.go b/support/db/mock_session.go index ce932cdbb3..e6cb58c987 100644 --- a/support/db/mock_session.go +++ b/support/db/mock_session.go @@ -72,14 +72,14 @@ func (m *MockSession) GetRaw(ctx context.Context, dest interface{}, query string return argss.Error(0) } -func (m *MockSession) Query(ctx context.Context, query squirrel.Sqlizer) (*sqlx.Rows, error) { +func (m *MockSession) Query(ctx context.Context, query squirrel.Sqlizer) (*Rows, error) { args := m.Called(ctx, query) - return args.Get(0).(*sqlx.Rows), args.Error(1) + return args.Get(0).(*Rows), args.Error(1) } -func (m *MockSession) QueryRaw(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) { +func (m *MockSession) QueryRaw(ctx context.Context, query string, args ...interface{}) (*Rows, error) { argss := m.Called(ctx, query, args) - return argss.Get(0).(*sqlx.Rows), argss.Error(1) + return argss.Get(0).(*Rows), argss.Error(1) } func (m *MockSession) Select(ctx context.Context, dest interface{}, query squirrel.Sqlizer) error { diff --git a/support/db/session.go b/support/db/session.go index 472fc40a37..6b5c2b18c0 100644 --- a/support/db/session.go +++ b/support/db/session.go @@ -12,28 +12,71 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/jmoiron/sqlx" "github.com/lib/pq" + "github.com/stellar/go/support/db/sqlutils" "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" ) +var DeadlineCtxKey = CtxKey("deadline") + +func noop() {} + +// context() checks if there is a override on the context timeout which is configured using DeadlineCtxKey. +// If the override exists, we return a new context with the desired deadline. Otherwise, we return the +// original context. +// Note that the override will not be applied if requestCtx has already been terminated. +// The timeout can be disabled by setting the DeadlineCtxKey value to a zero time.Time value, +// in that case the query will never be canceled. +func (s *Session) context(requestCtx context.Context) (context.Context, context.CancelFunc, error) { + var ctx context.Context + var cancel context.CancelFunc + + // if there is no DeadlineCtxKey value in the context we default to using the request context. + // this case is expected during ingestion where we don't want any queries to be canceled unless + // horizon is shutting down. + deadline, ok := requestCtx.Value(&DeadlineCtxKey).(time.Time) + if !ok { + return requestCtx, noop, nil + } + + // if requestCtx is already terminated don't proceed with the db statement + if requestCtx.Err() != nil { + return requestCtx, nil, requestCtx.Err() + } + + if deadline.IsZero() { + ctx, cancel = context.Background(), noop + } else { + ctx, cancel = context.WithDeadline(context.Background(), deadline) + } + return ctx, cancel, nil +} + // Begin binds this session to a new transaction. func (s *Session) Begin(ctx context.Context) error { if s.tx != nil { return errors.New("already in transaction") } + ctx, cancel, err := s.context(ctx) + if err != nil { + return err + } tx, err := s.DB.BeginTxx(ctx, nil) if err != nil { if knownErr := s.handleError(err, ctx); knownErr != nil { + cancel() return knownErr } + cancel() return errors.Wrap(err, "beginx failed") } log.Debug("sql: begin") s.tx = tx s.txOptions = nil + s.txCancel = cancel return nil } @@ -43,19 +86,26 @@ func (s *Session) BeginTx(ctx context.Context, opts *sql.TxOptions) error { if s.tx != nil { return errors.New("already in transaction") } + ctx, cancel, err := s.context(ctx) + if err != nil { + return err + } tx, err := s.DB.BeginTxx(ctx, opts) if err != nil { if knownErr := s.handleError(err, ctx); knownErr != nil { + cancel() return knownErr } + cancel() return errors.Wrap(err, "beginTx failed") } log.Debug("sql: begin") s.tx = tx s.txOptions = opts + s.txCancel = cancel return nil } @@ -93,6 +143,8 @@ func (s *Session) Commit() error { log.Debug("sql: commit") s.tx = nil s.txOptions = nil + s.txCancel() + s.txCancel = nil if knownErr := s.handleError(err, context.Background()); knownErr != nil { return knownErr @@ -135,7 +187,13 @@ func (s *Session) Get(ctx context.Context, dest interface{}, query sq.Sqlizer) e // GetRaw runs `query` with `args`, setting the first result found on // `dest`, if any. func (s *Session) GetRaw(ctx context.Context, dest interface{}, query string, args ...interface{}) error { - query, err := s.ReplacePlaceholders(query) + ctx, cancel, err := s.context(ctx) + if err != nil { + return err + } + defer cancel() + + query, err = s.ReplacePlaceholders(query) if err != nil { return errors.Wrap(err, "replace placeholders failed") } @@ -204,7 +262,13 @@ func (s *Session) ExecAll(ctx context.Context, script string) error { // ExecRaw runs `query` with `args` func (s *Session) ExecRaw(ctx context.Context, query string, args ...interface{}) (sql.Result, error) { - query, err := s.ReplacePlaceholders(query) + ctx, cancel, err := s.context(ctx) + if err != nil { + return nil, err + } + defer cancel() + + query, err = s.ReplacePlaceholders(query) if err != nil { return nil, errors.Wrap(err, "replace placeholders failed") } @@ -294,7 +358,7 @@ func (s *Session) handleError(dbErr error, ctx context.Context) error { } // Query runs `query`, returns a *sqlx.Rows instance -func (s *Session) Query(ctx context.Context, query sq.Sqlizer) (*sqlx.Rows, error) { +func (s *Session) Query(ctx context.Context, query sq.Sqlizer) (*Rows, error) { sql, args, err := s.build(query) if err != nil { return nil, err @@ -302,10 +366,26 @@ func (s *Session) Query(ctx context.Context, query sq.Sqlizer) (*sqlx.Rows, erro return s.QueryRaw(ctx, sql, args...) } +type Rows struct { + sqlx.Rows + cancel context.CancelFunc +} + +func (r *Rows) Close() error { + defer r.cancel() + return r.Rows.Close() +} + // QueryRaw runs `query` with `args` -func (s *Session) QueryRaw(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) { - query, err := s.ReplacePlaceholders(query) +func (s *Session) QueryRaw(ctx context.Context, query string, args ...interface{}) (*Rows, error) { + ctx, cancel, err := s.context(ctx) if err != nil { + return nil, err + } + + query, err = s.ReplacePlaceholders(query) + if err != nil { + cancel() return nil, errors.Wrap(err, "replace placeholders failed") } @@ -314,8 +394,12 @@ func (s *Session) QueryRaw(ctx context.Context, query string, args ...interface{ s.log(ctx, "query", start, query, args) if err == nil { - return result, nil + return &Rows{ + Rows: *result, + cancel: cancel, + }, nil } + defer cancel() if knownErr := s.handleError(err, ctx); knownErr != nil { return nil, knownErr @@ -350,6 +434,8 @@ func (s *Session) Rollback() error { log.Debug("sql: rollback") s.tx = nil s.txOptions = nil + s.txCancel() + s.txCancel = nil if knownErr := s.handleError(err, context.Background()); knownErr != nil { return knownErr @@ -381,8 +467,14 @@ func (s *Session) SelectRaw( query string, args ...interface{}, ) error { + ctx, cancel, err := s.context(ctx) + if err != nil { + return err + } + defer cancel() + s.clearSliceIfPossible(dest) - query, err := s.ReplacePlaceholders(query) + query, err = s.ReplacePlaceholders(query) if err != nil { return errors.Wrap(err, "replace placeholders failed") } diff --git a/support/db/session_test.go b/support/db/session_test.go index 1fd2a3902b..3ae06d2b4c 100644 --- a/support/db/session_test.go +++ b/support/db/session_test.go @@ -6,12 +6,12 @@ import ( "testing" "time" - //"github.com/lib/pq" "github.com/lib/pq" "github.com/prometheus/client_golang/prometheus" - "github.com/stellar/go/support/db/dbtest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/stellar/go/support/db/dbtest" ) func TestContextTimeoutDuringSql(t *testing.T) { @@ -140,6 +140,45 @@ func TestStatementTimeout(t *testing.T) { assertDbErrorMetrics(reg, "n/a", "57014", "statement_timeout", assert) } +func TestDeadlineOverride(t *testing.T) { + db := dbtest.Postgres(t).Load(testSchema) + defer db.Close() + + sess := &Session{DB: db.Open()} + defer sess.DB.Close() + + resultCtx, _, err := sess.context(context.Background()) + assert.NoError(t, err) + _, ok := resultCtx.Deadline() + assert.False(t, ok) + + deadline := time.Now().Add(time.Hour) + requestCtx := context.WithValue(context.Background(), &DeadlineCtxKey, deadline) + resultCtx, _, err = sess.context(requestCtx) + assert.NoError(t, err) + d, ok := resultCtx.Deadline() + assert.True(t, ok) + assert.Equal(t, deadline, d) + + requestCtx, cancel := context.WithDeadline(requestCtx, time.Now().Add(time.Minute*30)) + resultCtx, _, err = sess.context(requestCtx) + assert.NoError(t, err) + d, ok = resultCtx.Deadline() + assert.True(t, ok) + assert.Equal(t, deadline, d) + + cancel() + assert.NoError(t, resultCtx.Err()) + _, _, err = sess.context(requestCtx) + assert.EqualError(t, err, "context canceled") + + var emptyTime time.Time + resultCtx, _, err = sess.context(context.WithValue(context.Background(), &DeadlineCtxKey, emptyTime)) + assert.NoError(t, err) + _, ok = resultCtx.Deadline() + assert.False(t, ok) +} + func TestSession(t *testing.T) { db := dbtest.Postgres(t).Load(testSchema) defer db.Close() From 98afa8d126cbba95971bbaa8c391b78333badf49 Mon Sep 17 00:00:00 2001 From: tamirms Date: Thu, 14 Mar 2024 11:16:00 +0000 Subject: [PATCH 09/11] services/horizon/internal: Add limit on number of horizon requests in flight (#5244) --- services/horizon/internal/app.go | 1 + services/horizon/internal/config.go | 11 ++++++----- services/horizon/internal/flags.go | 14 ++++++++++++-- services/horizon/internal/httpx/router.go | 12 ++++++++---- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/services/horizon/internal/app.go b/services/horizon/internal/app.go index ed338f2ac3..8fb86b5f54 100644 --- a/services/horizon/internal/app.go +++ b/services/horizon/internal/app.go @@ -533,6 +533,7 @@ func (a *App) init() error { StaleThreshold: a.config.StaleThreshold, ConnectionTimeout: a.config.ConnectionTimeout, ClientQueryTimeout: a.config.ClientQueryTimeout, + MaxConcurrentRequests: a.config.MaxConcurrentRequests, MaxHTTPRequestSize: a.config.MaxHTTPRequestSize, NetworkPassphrase: a.config.NetworkPassphrase, MaxPathLength: a.config.MaxPathLength, diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go index eb89d72efd..94b6f5514b 100644 --- a/services/horizon/internal/config.go +++ b/services/horizon/internal/config.go @@ -40,11 +40,12 @@ type Config struct { ConnectionTimeout time.Duration ClientQueryTimeout time.Duration // MaxHTTPRequestSize is the maximum allowed request payload size - MaxHTTPRequestSize uint - RateQuota *throttled.RateQuota - FriendbotURL *url.URL - LogLevel logrus.Level - LogFile string + MaxHTTPRequestSize uint + RateQuota *throttled.RateQuota + MaxConcurrentRequests uint + FriendbotURL *url.URL + LogLevel logrus.Level + LogFile string // MaxPathLength is the maximum length of the path returned by `/paths` endpoint. MaxPathLength uint diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 48a83057f4..9df44af44d 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -69,8 +69,9 @@ const ( // StellarTestnet is a constant representing the Stellar test network StellarTestnet = "testnet" - defaultMaxHTTPRequestSize = uint(200 * 1024) - clientQueryTimeoutNotSet = -1 + defaultMaxConcurrentRequests = uint(1000) + defaultMaxHTTPRequestSize = uint(200 * 1024) + clientQueryTimeoutNotSet = -1 ) var ( @@ -471,6 +472,15 @@ func Flags() (*Config, support.ConfigOptions) { Usage: "sets the limit on the maximum allowed http request payload size, default is 200kb, to disable the limit check, set to 0, only do so if you acknowledge the implications of accepting unbounded http request payload sizes.", UsedInCommands: ApiServerCommands, }, + &support.ConfigOption{ + Name: "max-concurrent-requests", + ConfigKey: &config.MaxConcurrentRequests, + OptType: types.Uint, + FlagDefault: defaultMaxConcurrentRequests, + Usage: "sets the limit on the maximum number of concurrent http requests, default is 1000, to disable the limit set to 0. " + + "If Horizon receives a request which would exceed the limit of concurrent http requests, Horizon will respond with a 429 status code.", + UsedInCommands: ApiServerCommands, + }, &support.ConfigOption{ Name: "per-hour-rate-limit", ConfigKey: &config.RateQuota, diff --git a/services/horizon/internal/httpx/router.go b/services/horizon/internal/httpx/router.go index 60cf617468..4ba978a96a 100644 --- a/services/horizon/internal/httpx/router.go +++ b/services/horizon/internal/httpx/router.go @@ -28,10 +28,11 @@ import ( ) type RouterConfig struct { - DBSession db.SessionInterface - PrimaryDBSession db.SessionInterface - TxSubmitter *txsub.System - RateQuota *throttled.RateQuota + DBSession db.SessionInterface + PrimaryDBSession db.SessionInterface + TxSubmitter *txsub.System + RateQuota *throttled.RateQuota + MaxConcurrentRequests uint BehindCloudflare bool BehindAWSLoadBalancer bool @@ -91,6 +92,9 @@ func (r *Router) addMiddleware(config *RouterConfig, BehindAWSLoadBalancer: config.BehindAWSLoadBalancer, })) r.Use(loggerMiddleware(serverMetrics)) + if config.MaxConcurrentRequests > 0 { + r.Use(chimiddleware.Throttle(int(config.MaxConcurrentRequests))) + } r.Use(timeoutMiddleware(config.ConnectionTimeout)) if config.MaxHTTPRequestSize > 0 { r.Use(func(handler http.Handler) http.Handler { From 55a279ff012e4796062f73636ff100df3074f36d Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Thu, 14 Mar 2024 11:05:07 -0700 Subject: [PATCH 10/11] update changelog notes for latest on 2.29.0 --- services/horizon/CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 50c1407d26..003bab1d5a 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -12,6 +12,7 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). - Bumped go version to the latest (1.22.1) ([5232](https://github.com/stellar/go/pull/5232)) - Add metrics for ingestion loaders ([5209](https://github.com/stellar/go/pull/5209)). - Add metrics for http api requests in flight and requests received ([5240](https://github.com/stellar/go/pull/5240)). +- Add `MAX_CONCURRENT_REQUESTS`, defaults to 1000, limits the number of horizon api requests in flight ([5244](https://github.com/stellar/go/pull/5244)) ### Fixed - History archive access is more effective when you pass list of URLs to Horizon: they will now be accessed in a round-robin fashion, use alternative archives on errors, and intelligently back off ([5224](https://github.com/stellar/go/pull/5224)) @@ -19,7 +20,8 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). - Removed duplicate "Processed Ledger" log statement during resume state ([5152](https://github.com/stellar/go/pull/5152)) - Fixed incorrect duration for ingestion processor metric ([5216](https://github.com/stellar/go/pull/5216)) - Fixed sql performance on account transactions query ([5229](https://github.com/stellar/go/pull/5229)) - +- Fix bug in claimable balance change processor ([5246](https://github.com/stellar/go/pull/5246)) +- Delay canceling queries from client side when there's a statement / transaction timeout configured in postgres ([5223](https://github.com/stellar/go/pull/5223)) ### Breaking Changes - The Horizon API Transaction resource field in json `result_meta_xdr` is now optional and Horizon API will not emit the field when Horizon has been configured with `SKIP_TXMETA=true`, effectively null, otherwise if Horizon is configured with `SKIP_TXMETA=false` which is default, then the API Transaction field `result_meta_xdr` will remain present and populated with base64 encoded xdr [5228](https://github.com/stellar/go/pull/5228). From 4a00bc7e1afbb895dc669853c98af8f6f6663223 Mon Sep 17 00:00:00 2001 From: tamirms Date: Thu, 14 Mar 2024 21:45:17 +0000 Subject: [PATCH 11/11] fix flag usage for max-concurrent-requests (#5247) --- services/horizon/internal/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 9df44af44d..5f11389a5f 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -478,7 +478,7 @@ func Flags() (*Config, support.ConfigOptions) { OptType: types.Uint, FlagDefault: defaultMaxConcurrentRequests, Usage: "sets the limit on the maximum number of concurrent http requests, default is 1000, to disable the limit set to 0. " + - "If Horizon receives a request which would exceed the limit of concurrent http requests, Horizon will respond with a 429 status code.", + "If Horizon receives a request which would exceed the limit of concurrent http requests, Horizon will respond with a 503 status code.", UsedInCommands: ApiServerCommands, }, &support.ConfigOption{