pkg/sqlutil/pg: create package; expand env config

Author: jmank88

go.mod

@@ -3,6 +3,7 @@ module github.com/smartcontractkit/chainlink-common
 go 1.21
 
 require (
+	github.com/XSAM/otelsql v0.29.0
 	github.com/confluentinc/confluent-kafka-go/v2 v2.3.0
 	github.com/dominikbraun/graph v0.23.0
 	github.com/fxamacker/cbor/v2 v2.5.0
@@ -14,6 +15,7 @@ require (
 	github.com/hashicorp/go-hclog v1.5.0
 	github.com/hashicorp/go-plugin v1.6.0
 	github.com/invopop/jsonschema v0.12.0
+	github.com/jackc/pgx/v4 v4.18.3
 	github.com/jmoiron/sqlx v1.4.0
 	github.com/jonboulle/clockwork v0.4.0
 	github.com/jpillora/backoff v1.0.0
@@ -24,6 +26,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.0
 	github.com/prometheus/client_golang v1.17.0
 	github.com/riferrei/srclient v0.5.4
+	github.com/scylladb/go-reflectx v1.0.1
 	github.com/shopspring/decimal v1.4.0
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c
 	github.com/stretchr/testify v1.9.0
@@ -42,6 +45,16 @@ require (
 	sigs.k8s.io/yaml v1.4.0
 )
 
+require (
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.14.3 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.3 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgtype v1.14.0 // indirect
+)
+
 require (
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect

go.sum

@@ -0,0 +1,49 @@
+package static
+
+import (
+	"os"
+	"runtime/debug"
+)
+
+// Unset is a sentinel value.
+const Unset = "unset"
+
+// Version and Checksum are set at compile time via build arguments.
+var (
+	// Program is updated to the full main program path if [debug.BuildInfo] is available.
+	Program = os.Args[0]
+	// Version is the semantic version of the build or Unset.
+	Version = Unset
+	// Checksum is the commit hash of the build or Unset.
+	Checksum = Unset
+)
+
+func init() {
+	buildInfo, ok := debug.ReadBuildInfo()
+	if ok {
+		Program = buildInfo.Main.Path
+		if Version == "" {
+			Version = buildInfo.Main.Version
+		}
+		if Checksum == "" {
+			Checksum = buildInfo.Main.Sum
+		}
+	}
+}
+
+// Short returns a 7-character sha prefix and version, or Unset if blank.
+func Short() (shaPre string, ver string) {
+	return short(Checksum, Version)
+}
+
+func short(sha, ver string) (string, string) {
+	if sha == "" {
+		sha = Unset
+	} else if len(sha) > 7 {
+		sha = sha[:7]
+	}
+	if ver == "" {
+		ver = Unset
+	}
+	return sha, ver
+}

pkg/config/static/static.go

@@ -1,13 +1,16 @@
 package logger
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"go.uber.org/zap/zaptest"
 	"go.uber.org/zap/zaptest/observer"
+
+	"github.com/smartcontractkit/chainlink-common/pkg/config/static"
 )
 
 // Logger is a minimal subset of smartcontractkit/chainlink/core/logger.Logger implemented by go.uber.org/zap.SugaredLogger
@@ -51,9 +54,17 @@ func New() (Logger, error) { return defaultConfig.New() }
 func (c *Config) New() (Logger, error) {
 	return NewWith(func(cfg *zap.Config) {
 		cfg.Level.SetLevel(c.Level)
+		cfg.InitialFields = map[string]interface{}{
+			"version": staticShortVerSha(),
+		}
 	})
 }
 
+func staticShortVerSha() string {
+	sha, ver := static.Short() //TODO rename?
+	return fmt.Sprintf("%s@%s", ver, sha)
+}
+
 // NewWith returns a new Logger from a modified [zap.Config].
 func NewWith(cfgFn func(*zap.Config)) (Logger, error) {
 	cfg := zap.NewProductionConfig()
@@ -76,7 +87,7 @@ func Test(tb testing.TB) Logger {
 			zapcore.DebugLevel,
 		),
 	)
-	return &logger{lggr.Sugar()}
+	return &logger{lggr.With(zap.String("version", staticShortVerSha())).Sugar()}
 }
 
 // TestSugared returns a new test SugaredLogger.

pkg/logger/logger.go

@@ -6,12 +6,20 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/go-plugin"
 )
 
 const (
-	envDatabaseURL            = "CL_DATABASE_URL"
+	envDatabaseURL                    = "CL_DATABASE_URL"
+	envDatabaseIdleInTxSessionTimeout = "CL_DATABASE_IDLE_IN_TX_SESSION_TIMEOUT"
+	envDatabaseLockTimeout            = "CL_DATABASE_LOCK_TIMEOUT"
+	envDatabaseQueryTimeout           = "CL_DATABASE_QUERY_TIMEOUT"
+	envDatabaseLogSQL                 = "CL_DATABASE_LOG_SQL"
+	envDatabaseMaxOpenConns           = "CL_DATABASE_MAX_OPEN_CONNS"
+	envDatabaseMaxIdleConns           = "CL_DATABASE_MAX_IDLE_CONNS"
+
 	envPromPort               = "CL_PROMETHEUS_PORT"
 	envTracingEnabled         = "CL_TRACING_ENABLED"
 	envTracingCollectorTarget = "CL_TRACING_COLLECTOR_TARGET"
@@ -23,7 +31,13 @@ const (
 // EnvConfig is the configuration between the application and the LOOP executable. The values
 // are fully resolved and static and passed via the environment.
 type EnvConfig struct {
-	DatabaseURL *url.URL
+	DatabaseURL                    *url.URL
+	DatabaseIdleInTxSessionTimeout time.Duration
+	DatabaseLockTimeout            time.Duration
+	DatabaseQueryTimeout           time.Duration
+	DatabaseLogSQL                 bool
+	DatabaseMaxOpenConns           int
+	DatabaseMaxIdleConns           int
 
 	PrometheusPort int
 
@@ -47,6 +61,12 @@ func (e *EnvConfig) AsCmdEnv() (env []string) {
 	// DatabaseURL is optional
 	if e.DatabaseURL != nil {
 		injectEnv[envDatabaseURL] = e.DatabaseURL.String()
+		injectEnv[envDatabaseIdleInTxSessionTimeout] = e.DatabaseIdleInTxSessionTimeout.String()
+		injectEnv[envDatabaseLockTimeout] = e.DatabaseLockTimeout.String()
+		injectEnv[envDatabaseQueryTimeout] = e.DatabaseQueryTimeout.String()
+		injectEnv[envDatabaseLogSQL] = strconv.FormatBool(e.DatabaseLogSQL)
+		injectEnv[envDatabaseMaxOpenConns] = strconv.Itoa(e.DatabaseMaxOpenConns)
+		injectEnv[envDatabaseMaxIdleConns] = strconv.Itoa(e.DatabaseMaxIdleConns)
 	}
 
 	for k, v := range e.TracingAttributes {
@@ -61,13 +81,39 @@ func (e *EnvConfig) AsCmdEnv() (env []string) {
 
 // parse deserializes environment variables
 func (e *EnvConfig) parse() error {
-	promPortStr := os.Getenv(envPromPort)
 	var err error
 	e.DatabaseURL, err = getDatabaseURL()
 	if err != nil {
-		return fmt.Errorf("failed to parse %s: %q", envDatabaseURL, err)
+		return err
+	}
+	if e.DatabaseURL != nil {
+		e.DatabaseIdleInTxSessionTimeout, err = getDatabaseIdleInTxSessionTimeout()
+		if err != nil {
+			return err
+		}
+		e.DatabaseLockTimeout, err = getDatabaseLockTimeout()
+		if err != nil {
+			return err
+		}
+		e.DatabaseQueryTimeout, err = getDatabaseQueryTimeout()
+		if err != nil {
+			return err
+		}
+		e.DatabaseLogSQL, err = getDatabaseLogSQL()
+		if err != nil {
+			return err
+		}
+		e.DatabaseMaxOpenConns, err = getDatabaseMaxOpenConns()
+		if err != nil {
+			return err
+		}
+		e.DatabaseMaxIdleConns, err = getDatabaseMaxIdleConns()
+		if err != nil {
+			return err
+		}
 	}
 
+	promPortStr := os.Getenv(envPromPort)
 	e.PrometheusPort, err = strconv.Atoi(promPortStr)
 	if err != nil {
 		return fmt.Errorf("failed to parse %s = %q: %w", envPromPort, promPortStr, err)
@@ -164,3 +210,27 @@ func getDatabaseURL() (*url.URL, error) {
 	}
 	return u, nil
 }
+
+func getDatabaseIdleInTxSessionTimeout() (time.Duration, error) {
+	return time.ParseDuration(os.Getenv(envDatabaseIdleInTxSessionTimeout))
+}
+
+func getDatabaseLockTimeout() (time.Duration, error) {
+	return time.ParseDuration(os.Getenv(envDatabaseLockTimeout))
+}
+
+func getDatabaseQueryTimeout() (time.Duration, error) {
+	return time.ParseDuration(os.Getenv(envDatabaseQueryTimeout))
+}
+
+func getDatabaseLogSQL() (bool, error) {
+	return strconv.ParseBool(os.Getenv(envDatabaseLogSQL))
+}
+
+func getDatabaseMaxOpenConns() (int, error) {
+	return strconv.Atoi(os.Getenv(envDatabaseMaxOpenConns))
+}
+
+func getDatabaseMaxIdleConns() (int, error) {
+	return strconv.Atoi(os.Getenv(envDatabaseMaxIdleConns))
+}

pkg/loop/config.go

@@ -5,6 +5,7 @@ import (
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/go-plugin"
 	"github.com/stretchr/testify/assert"
@@ -16,10 +17,18 @@ import (
 
 func TestEnvConfig_parse(t *testing.T) {
 	cases := []struct {
-		name                           string
-		envVars                        map[string]string
-		expectError                    bool
-		expectedDatabaseURL            string
+		name        string
+		envVars     map[string]string
+		expectError bool
+
+		expectedDatabaseURL                    string
+		expectedDatabaseIdleInTxSessionTimeout time.Duration
+		expectedDatabaseLockTimeout            time.Duration
+		expectedDatabaseQueryTimeout           time.Duration
+		expectedDatabaseLogSQL                 bool
+		expectedDatabaseMaxOpenConns           int
+		expectedDatabaseMaxIdleConns           int
+
 		expectedPrometheusPort         int
 		expectedTracingEnabled         bool
 		expectedTracingCollectorTarget string
@@ -29,16 +38,31 @@ func TestEnvConfig_parse(t *testing.T) {
 		{
 			name: "All variables set correctly",
 			envVars: map[string]string{
-				envDatabaseURL:              "postgres://user:password@localhost:5432/db",
+				envDatabaseURL:                    "postgres://user:password@localhost:5432/db",
+				envDatabaseIdleInTxSessionTimeout: "42s",
+				envDatabaseLockTimeout:            "8m",
+				envDatabaseQueryTimeout:           "7s",
+				envDatabaseLogSQL:                 "true",
+				envDatabaseMaxOpenConns:           "9999",
+				envDatabaseMaxIdleConns:           "8080",
+
 				envPromPort:                 "8080",
 				envTracingEnabled:           "true",
 				envTracingCollectorTarget:   "some:target",
 				envTracingSamplingRatio:     "1.0",
 				envTracingTLSCertPath:       "internal/test/fixtures/client.pem",
 				envTracingAttribute + "XYZ": "value",
 			},
-			expectError:                    false,
-			expectedDatabaseURL:            "postgres://user:password@localhost:5432/db",
+			expectError: false,
+
+			expectedDatabaseURL:                    "postgres://user:password@localhost:5432/db",
+			expectedDatabaseIdleInTxSessionTimeout: 42 * time.Second,
+			expectedDatabaseLockTimeout:            8 * time.Minute,
+			expectedDatabaseQueryTimeout:           7 * time.Second,
+			expectedDatabaseLogSQL:                 true,
+			expectedDatabaseMaxOpenConns:           9999,
+			expectedDatabaseMaxIdleConns:           8080,
+
 			expectedPrometheusPort:         8080,
 			expectedTracingEnabled:         true,
 			expectedTracingCollectorTarget: "some:target",
@@ -89,6 +113,25 @@ func TestEnvConfig_parse(t *testing.T) {
 					if config.DatabaseURL.String() != tc.expectedDatabaseURL {
 						t.Errorf("Expected Database URL %s, got %s", tc.expectedDatabaseURL, config.DatabaseURL)
 					}
+					if config.DatabaseIdleInTxSessionTimeout != tc.expectedDatabaseIdleInTxSessionTimeout {
+						t.Errorf("Expected Database idle in tx session timeout %s, got %s", tc.expectedDatabaseIdleInTxSessionTimeout, config.DatabaseIdleInTxSessionTimeout)
+					}
+					if config.DatabaseLockTimeout != tc.expectedDatabaseLockTimeout {
+						t.Errorf("Expected Database lock timeout %s, got %s", tc.expectedDatabaseLockTimeout, config.DatabaseLockTimeout)
+					}
+					if config.DatabaseQueryTimeout != tc.expectedDatabaseQueryTimeout {
+						t.Errorf("Expected Database query timeout %s, got %s", tc.expectedDatabaseQueryTimeout, config.DatabaseQueryTimeout)
+					}
+					if config.DatabaseLogSQL != tc.expectedDatabaseLogSQL {
+						t.Errorf("Expected Database log sql %t, got %t", tc.expectedDatabaseLogSQL, config.DatabaseLogSQL)
+					}
+					if config.DatabaseMaxOpenConns != tc.expectedDatabaseMaxOpenConns {
+						t.Errorf("Expected Database max open conns %d, got %d", tc.expectedDatabaseMaxOpenConns, config.DatabaseMaxOpenConns)
+					}
+					if config.DatabaseMaxIdleConns != tc.expectedDatabaseMaxIdleConns {
+						t.Errorf("Expected Database max idle conns %d, got %d", tc.expectedDatabaseMaxIdleConns, config.DatabaseMaxIdleConns)
+					}
+
 					if config.PrometheusPort != tc.expectedPrometheusPort {
 						t.Errorf("Expected Prometheus port %d, got %d", tc.expectedPrometheusPort, config.PrometheusPort)
 					}