Skip to content

Commit

Permalink
pooler and worklow
Browse files Browse the repository at this point in the history
  • Loading branch information
Schmaetz committed Jun 12, 2024
1 parent a92bf7e commit 9e37086
Show file tree
Hide file tree
Showing 8 changed files with 314 additions and 25 deletions.
26 changes: 26 additions & 0 deletions .github/workflows/build_containers.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
name: Build and publish the Containers

on:
push:
branches:
- build_workflow

jobs:
build:
runs-on: ubuntu-latest

steps:
- name: Checkout repository
uses: actions/checkout@v2

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1

- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: '1.22'

- name: Build base
run: |
make base
16 changes: 16 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ postgres: base postgres
postgres-stage: base postgres-stage
postgres-gis: base postgres-gis
postgres-oracle: base postgres-oracle
pgbouncer: pgbouncer
exporter: exporter

base-build:
Expand Down Expand Up @@ -136,6 +137,21 @@ postgres-oracle-build:

postgres-oracle: postgres-oracle-build

pgbouncer-build:
docker build $(ROOTPATH) \
--file $(ROOTPATH)/docker/pgbouncer/Dockerfile \
--tag cybertec-pg-container/pgbouncer:$(IMAGE_TAG)-$(BETA)$(BUILD) \
--build-arg BASE_IMAGE=$(BASE_IMAGE) \
--build-arg CONTAINERIMAGE=${CONTAINERIMAGE} \
--build-arg IMAGE_REPOSITORY=$(IMAGE_REPOSITORY) \
--build-arg BASEOS=$(BASEOS) \
--build-arg PACKAGER=$(PACKAGER) \
--build-arg CONTAINERSUITE=$(CONTAINERSUITE) \
--build-arg BUILD=$(BUILD) \
--build-arg PGVERSION=$(PGVERSION)

pgbouncer: pgbouncer-build

exporter-build:
docker build $(ROOTPATH) \
--file $(ROOTPATH)/docker/exporter/Dockerfile \
Expand Down
36 changes: 36 additions & 0 deletions docker/pgbouncer/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
ARG CONTAINERSUITE
ARG BUILD
ARG BASEOS
ARG CONTAINERIMAGE

FROM ${CONTAINERSUITE}/base:${BASEOS}-${BUILD} AS builder

ARG PACKAGER

RUN ${PACKAGER} -y install --nodocs \
--setopt=skip_missing_names_on_install=False \
git \
go \
dumb-init \
pgbouncer \
&& ${PACKAGER} -y clean all ;


# FROM ${CONTAINERIMAGE}
# COPY --from=builder /usr/bin/dumb-init /usr/bin/dumb-init
# COPY --from=builder ./postgres_exporter/postgres_exporter /bin/postgres_exporter

COPY launcher/pgbouncer/launch.sh /
COPY scripts/pgbouncer/* /etc/pgbouncer/
# add pgbackrest-common files
ADD /scripts/nss_wrapper/nss_wrapper_pgbouncer.sh /scripts/nss_wrapper/nss_wrapper_pgbouncer.sh

RUN mkdir -p /etc/pgbouncer/certs && chown -R pgbouncer:pgbouncer /etc/pgbouncer

# EXPOSE 9187

ENTRYPOINT ["/scripts/nss_wrapper/nss_wrapper_pgbouncer.sh"]

USER 996

CMD ["/bin/sh", "/launch.sh", "init"]
31 changes: 31 additions & 0 deletions launcher/pgbouncer/launch.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/bash
set -ex

if [ "$PGUSER" = "postgres" ]; then
echo "WARNING: pgbouncer will connect with a superuser privileges!"
echo "You need to fix this as soon as possible."
fi

if [ -z "${CONNECTION_POOLER_CLIENT_TLS_CRT}" ]; then
openssl req -nodes -new -x509 -subj /CN=spilo.dummy.org \
-keyout /etc/pgbouncer/certs/pgbouncer.key \
-out /etc/pgbouncer/certs/pgbouncer.crt
# -keyout /etc/ssl/certs/pgbouncer.key \
# -out /etc/ssl/certs/pgbouncer.crt
else
ln -s ${CONNECTION_POOLER_CLIENT_TLS_CRT} /etc/pgbouncer/certs/pgbouncer.crt
ln -s ${CONNECTION_POOLER_CLIENT_TLS_KEY} /etc/pgbouncer/certs/pgbouncer.key
if [ ! -z "${CONNECTION_POOLER_CLIENT_CA_FILE}" ]; then
ln -s ${CONNECTION_POOLER_CLIENT_CA_FILE} /etc/pgbouncer/certs/ca.crt
fi
# ln -s ${CONNECTION_POOLER_CLIENT_TLS_CRT} /etc/ssl/certs/pgbouncer.crt
# ln -s ${CONNECTION_POOLER_CLIENT_TLS_KEY} /etc/ssl/certs/pgbouncer.key
# if [ ! -z "${CONNECTION_POOLER_CLIENT_CA_FILE}" ]; then
# ln -s ${CONNECTION_POOLER_CLIENT_CA_FILE} /etc/ssl/certs/ca.crt
# fi
fi

envsubst < /etc/pgbouncer/pgbouncer.ini.tmpl > /etc/pgbouncer/pgbouncer.ini
envsubst < /etc/pgbouncer/auth_file.txt.tmpl > /etc/pgbouncer/auth_file.txt

exec /bin/pgbouncer /etc/pgbouncer/pgbouncer.ini
117 changes: 92 additions & 25 deletions scripts/exporter/queries/queries_nodemx.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,22 +28,54 @@ cpo_pgnodemx_process:

cpo_pgnodemx_mem:
query: "
with d(key, val) as
(select key, val from exporter.cgroup_setof_kv('memory.stat'))
WITH d AS (
SELECT key, val
FROM exporter.cgroup_setof_kv('memory.stat')
),
limits AS (
SELECT
CASE
WHEN exporter.cgroup_mode() = 'legacy' AND exporter.cgroup_scalar_bigint('memory.limit_in_bytes') = 9223372036854771712 THEN 0
WHEN exporter.cgroup_mode() = 'legacy' THEN exporter.cgroup_scalar_bigint('memory.limit_in_bytes')
WHEN exporter.cgroup_scalar_bigint('memory.max') = 9223372036854775807 THEN 0
ELSE exporter.cgroup_scalar_bigint('memory.max')
END AS limit,
exporter.cgroup_mode() AS mode
),
values AS (
SELECT
(SELECT val FROM d WHERE key = 'cache') AS cache,
(SELECT val FROM d WHERE key = 'rss') AS rss,
(SELECT val FROM d WHERE key = 'shmem') AS shmem,
(SELECT val FROM d WHERE key = 'mapped_file') AS mapped_file,
(SELECT val FROM d WHERE key = 'dirty') AS dirty_legacy,
(SELECT val FROM d WHERE key = 'file_dirty') AS dirty,
(SELECT val FROM d WHERE key = 'active_anon') AS active_anon,
(SELECT val FROM d WHERE key = 'inactive_anon') AS inactive_anon,
(SELECT val FROM d WHERE key = 'active_file') AS active_file,
(SELECT val FROM d WHERE key = 'inactive_file') AS inactive_file
)
SELECT
exporter.kdapi_scalar_bigint('mem_request') as request,
case when exporter.cgroup_scalar_bigint('memory.limit_in_bytes') = 9223372036854771712 then 0 else exporter.cgroup_scalar_bigint('memory.limit_in_bytes') end as limit,
(select val from d where key='cache') as cache,
(select val from d where key='rss') as rss,
(select val from d where key='shmem') as shmem,
(select val from d where key='mapped_file') as mapped_file,
(select val from d where key='dirty') as dirty,
(select val from d where key='active_anon') as active_anon,
(select val from d where key='inactive_anon') as inactive_anon,
(select val from d where key='active_file') as active_file,
(select val from d where key='inactive_file') as inactive_file,
exporter.cgroup_scalar_bigint('memory.usage_in_bytes') as usage_in_bytes,
exporter.cgroup_scalar_bigint('memory.kmem.usage_in_bytes') as kmem_usage_in_bytes
exporter.kdapi_scalar_bigint('mem_request') AS request,
limits.limit,
CASE WHEN limits.mode = 'legacy' THEN values.cache ELSE 0 END AS cache,
CASE WHEN limits.mode = 'legacy' THEN values.rss ELSE 0 END AS rss,
values.shmem,
CASE WHEN limits.mode = 'legacy' THEN values.mapped_file ELSE 0 END AS mapped_file,
CASE WHEN limits.mode = 'legacy' THEN values.dirty_legacy ELSE values.dirty END AS dirty,
values.active_anon,
values.inactive_anon,
values.active_file,
values.inactive_file,
CASE
WHEN limits.mode = 'legacy' THEN exporter.cgroup_scalar_bigint('memory.usage_in_bytes')
ELSE exporter.cgroup_scalar_bigint('memory.current')
END AS usage_in_bytes,
CASE
WHEN limits.mode = 'legacy' THEN exporter.cgroup_scalar_bigint('memory.kmem.usage_in_bytes')
ELSE 0
END AS kmem_usage_in_byte
FROM limits, values
"
metrics:
- request:
Expand Down Expand Up @@ -98,8 +130,25 @@ cpo_pgnodemx_cpu:
description: "CPU limit value in milli cores"

cpo_pgnodemx_cpucfs:
query: "SELECT exporter.cgroup_scalar_bigint('cpu.cfs_period_us') as period_us,
case when exporter.cgroup_scalar_bigint('cpu.cfs_quota_us') < 0 then 0 else exporter.cgroup_scalar_bigint('cpu.cfs_quota_us') end as quota_us"
query: "WITH cgroup_mode AS (
SELECT exporter.cgroup_mode() AS mode
),
cpu_values AS (
SELECT
exporter.cgroup_scalar_bigint('cpu.cfs_period_us') AS cfs_period_us,
exporter.cgroup_scalar_bigint('cpu.cfs_quota_us') AS cfs_quota_us,
exporter.cgroup_array_bigint('cpu.max') AS cpu_max
)
SELECT
CASE
WHEN cm.mode = 'legacy' THEN cv.cfs_period_us
ELSE (cv.cpu_max)[2]
END AS period_us,
CASE
WHEN cm.mode = 'legacy' THEN GREATEST(cv.cfs_quota_us, 0)
ELSE GREATEST((cv.cpu_max)[1], 0)
END AS quota_us
FROM cgroup_mode cm, cpu_values cv"
metrics:
- period_us:
usage: "GAUGE"
Expand All @@ -109,7 +158,19 @@ cpo_pgnodemx_cpucfs:
description: "the length of a period (in microseconds)"

cpo_pgnodemx_cpuacct:
query: "SELECT exporter.cgroup_scalar_bigint('cpuacct.usage') as usage, clock_timestamp() as usage_ts"
query: "WITH cpu_usage AS (
SELECT
exporter.cgroup_mode() AS mode,
exporter.cgroup_scalar_bigint('cpuacct.usage') AS legacy_usage,
(SELECT val FROM exporter.cgroup_setof_kv('cpu.stat') WHERE key = 'usage_usec') * 1000 AS v2_usage
)
SELECT
CASE
WHEN cu.mode = 'legacy' THEN cu.legacy_usage
ELSE cu.v2_usage
END AS usage,
clock_timestamp() AS usage_ts
FROM cpu_usage cu"
metrics:
- usage:
usage: "GAUGE"
Expand All @@ -119,22 +180,28 @@ cpo_pgnodemx_cpuacct:
description: "CPU usage snapshot timestamp"

cpo_pgnodemx_cpustat:
query: "WITH d(key, val) AS
(select key, val from exporter.cgroup_setof_kv('cpu.stat'))
query: "WITH d AS (
SELECT key, val, exporter.cgroup_mode() AS mode
FROM exporter.cgroup_setof_kv('cpu.stat')
)
SELECT
(SELECT val FROM d WHERE key='nr_periods') AS nr_periods,
(SELECT val FROM d WHERE key='nr_throttled') AS nr_throttled,
(SELECT val FROM d WHERE key='throttled_time') AS throttled_time, clock_timestamp() as snap_ts"
MAX(CASE WHEN key = 'nr_periods' THEN val END) AS nr_periods,
MAX(CASE WHEN key = 'nr_throttled' THEN val END) AS nr_throttled,
MAX(CASE WHEN mode = 'legacy' AND key = 'throttled_usec' THEN val
WHEN mode = 'unified' AND key = 'throttled_usec' THEN val*1000 END) AS throttled_time_in_nanoseconds,
clock_timestamp() AS snap_ts
FROM d
"
metrics:
- nr_periods:
usage: "GAUGE"
description: "number of periods that any thread was runnable"
- nr_throttled:
usage: "GAUGE"
description: "number of runnable periods in which the application used its entire quota and was throttled"
- throttled_time:
- throttled_time_in_nanoseconds:
usage: "GAUGE"
description: "sum total amount of time individual threads within the exporter.cgroup were throttled"
description: "sum total amount of time in nanoseconds individual threads within the exporter.cgroup were throttled"
- snap_ts:
usage: "GAUGE"
description: "CPU stat snapshot timestamp"
Expand Down
46 changes: 46 additions & 0 deletions scripts/nss_wrapper/nss_wrapper_pgbouncer.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/bin/bash

# Define some needed ENVs & Variables
export NSS_USERNAME=${NSS_USERNAME:-'pgbouncer'}
export NSS_USERDESC=${NSS_USERDESC:-'pgbouncer'}
export CURRENT_USER=$(id -u)
export CURRENT_GROUP=$(id -g)

# Prepare Folders and Files
NSS_ROOT_DIR="/tmp/nss_wrapper"
NSS_PASSWD="${NSS_ROOT_DIR}/passwd"
NSS_GROUP="${NSS_ROOT_DIR}//group"

mkdir -p ${NSS_ROOT_DIR}
chmod g+rwx ${NSS_ROOT_DIR}

[[ -f "${NSS_PASSWD}" ]] || cp "/etc/passwd" "${NSS_PASSWD}"
[[ -f "${NSS_GROUP}" ]] || cp "/etc/group" "${NSS_GROUP}"

# Check if User and Group already exists, if not add it
if [[ ! $(cat "${NSS_PASSWD}") =~ ${NSS_USERNAME}:x:${CURRENT_USER} ]]; then
passwd_tmp="${NSS_WRAPPER_DIR}/passwd_tmp"
cp "${NSS_PASSWD}" "${NSS_PASSWD}.tmp"
sed -i "/${NSS_USERNAME}:x:/d" "${NSS_PASSWD}.tmp"
sed -i "/${CURRENT_USER}:x:/d" "${NSS_PASSWD}.tmp"
echo '${NSS_USERNAME}:x:${CURRENT_USER}:${CURRENT_GROUP}:${NSS_USERDESC}:${HOME}:/bin/bash\n' >> "${NSS_PASSWD}.tmp"
envsubst < "${NSS_PASSWD}.tmp" > "${NSS_PASSWD}"
rm "${NSS_PASSWD}.tmp"
echo "User was added via nss_wrapper"
fi

if [[ ! $(cat "${NSS_GROUP}") =~ ${NSS_USERNAME}:x:${CURRENT_USER} ]]; then
cp "${NSS_GROUP}" "${NSS_GROUP}.tmp"
sed -i "/${NSS_USERNAME}:x:/d" "${NSS_GROUP}.tmp"
printf '${NSS_USERNAME}:x:${CURRENT_USER}:${NSS_USERNAME}\n' >> "${NSS_GROUP}.tmp"
envsubst < "${NSS_GROUP}.tmp" > "${NSS_GROUP}"
rm "${NSS_GROUP}.tmp"
echo "Group was added via nss_wrapper"
fi

export LD_PRELOAD=/usr/lib64/libnss_wrapper.so
export NSS_WRAPPER_PASSWD="${NSS_PASSWD}"
export NSS_WRAPPER_GROUP="${NSS_GROUP}"

exec "$@"

1 change: 1 addition & 0 deletions scripts/pgbouncer/auth_file.txt.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"$PGUSER" "$PGPASSWORD"
66 changes: 66 additions & 0 deletions scripts/pgbouncer/pgbouncer.ini.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# vim: set ft=dosini:

[databases]
* = host=$PGHOST port=$PGPORT auth_user=$PGUSER
postgres = host=$PGHOST port=$PGPORT auth_user=$PGUSER

[pgbouncer]
pool_mode = $CONNECTION_POOLER_MODE
listen_port = $CONNECTION_POOLER_PORT
listen_addr = *
auth_type = scram-sha-256
auth_file = /etc/pgbouncer/auth_file.txt
auth_dbname = postgres
admin_users = $PGUSER
stats_users_prefix = robot_
auth_query = SELECT * FROM $PGSCHEMA.user_lookup($1)
logfile = /var/log/pgbouncer/pgbouncer.log
pidfile = /var/run/pgbouncer/pgbouncer.pid

server_tls_sslmode = require
server_tls_ca_file = /etc/pgbouncer/certs/pgbouncer.crt
server_tls_protocols = secure
client_tls_sslmode = require
client_tls_key_file = /etc/pgbouncer/certs/pgbouncer.key
client_tls_cert_file = /etc/pgbouncer/certs/pgbouncer.crt

log_connections = 0
log_disconnections = 0

# How many server connections to allow per user/database pair.
default_pool_size = $CONNECTION_POOLER_DEFAULT_SIZE

# Add more server connections to pool if below this number. Improves behavior
# when usual load comes suddenly back after period of total inactivity.
#
# NOTE: This value is per pool, i.e. a pair of (db, user), not a global one.
# Which means on the higher level it has to be calculated from the max allowed
# database connections and number of databases and users. If not taken into
# account, then for too many users or databases PgBouncer will go crazy
# opening/evicting connections. For now disable it.
#
# min_pool_size = $CONNECTION_POOLER_MIN_SIZE

# How many additional connections to allow to a pool
reserve_pool_size = $CONNECTION_POOLER_RESERVE_SIZE

# Maximum number of client connections allowed.
max_client_conn = $CONNECTION_POOLER_MAX_CLIENT_CONN

# Do not allow more than this many connections per database (regardless of
# pool, i.e. user)
max_db_connections = $CONNECTION_POOLER_MAX_DB_CONN

# If a client has been in "idle in transaction" state longer, it will be
# disconnected. [seconds]
idle_transaction_timeout = 600

# If login failed, because of failure from connect() or authentication that
# pooler waits this much before retrying to connect. Default is 15. [seconds]
server_login_retry = 5

# To ignore extra parameter in startup packet. By default only 'database' and
# 'user' are allowed, all others raise error. This is needed to tolerate
# overenthusiastic JDBC wanting to unconditionally set 'extra_float_digits=2'
# in startup packet.
ignore_startup_parameters = extra_float_digits,options

0 comments on commit 9e37086

Please sign in to comment.