Skip to content

Commit 1be8bf5

Browse files
author
Mohit Joshi
committed
Added PG TDE tests
1 parent cd9e508 commit 1be8bf5

File tree

8 files changed

+234
-215
lines changed

8 files changed

+234
-215
lines changed

‎postgresql/bugs/PG-1727/PG-1727.sh

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,12 @@ PRIMARY_DATA=$INSTALL_DIR/primary_data
66
REPLICA_DATA=$INSTALL_DIR/replica_data
77
PRIMARY_LOGFILE=$PRIMARY_DATA/server.log
88
REPLICA_LOGFILE=$REPLICA_DATA/server.log
9-
TABLES=1000
9+
TABLES=100
10+
export PGCTLTIMEOUT=600
1011

1112
# initate the database
1213
initialize_server() {
13-
PG_PIDS=$(lsof -ti :5433 -ti :5434 2>/dev/null) || true
14+
PG_PIDS=$(lsof -ti :5433 -ti :5434 2>/dev/null)
1415
if [[ -n "$PG_PIDS" ]]; then
1516
echo "Killing PostgreSQL processes: $PG_PIDS"
1617
kill -9 $PG_PIDS
@@ -59,9 +60,9 @@ logging_collector = on
5960
log_directory = '$REPLICA_DATA'
6061
log_filename = 'server.log'
6162
log_statement = 'all'
63+
unix_socket_directories = '/tmp'
6264
SQL
6365
$INSTALL_DIR/bin/pg_ctl -D $REPLICA_DATA -l $REPLICA_LOGFILE start
64-
6566
}
6667

6768
enable_tde_and_create_load() {
@@ -85,7 +86,7 @@ rotate_server_key() {
8586
echo "Rotating server keys for 30 seconds..."
8687

8788
local start_time=$(date +%s)
88-
local duration=30
89+
local duration=8
8990

9091
while [[ $(($(date +%s) - start_time)) -lt $duration ]]; do
9192
RAND=$RANDOM
@@ -97,11 +98,11 @@ rotate_server_key() {
9798

9899
verify_streaming_replication() {
99100
echo "Creating verification table on primary..."
100-
$INSTALL_DIR/bin/psql -d postgres -p 5433 -c "CREATE TABLE verify_replication(id INT PRIMARY KEY, val TEXT);" || exit 1
101-
$INSTALL_DIR/bin/psql -d postgres -p 5433 -c "INSERT INTO verify_replication VALUES (1, 'streaming_test');" || exit 1
101+
$INSTALL_DIR/bin/psql -d postgres -p 5433 -c "CREATE TABLE verify_replication(id INT PRIMARY KEY, val TEXT);"
102+
$INSTALL_DIR/bin/psql -d postgres -p 5433 -c "INSERT INTO verify_replication VALUES (1, 'streaming_test');"
102103

103104
echo "Waiting for replication to apply..."
104-
sleep 5
105+
sleep 20
105106

106107
echo "Checking data on replica..."
107108
result=$($INSTALL_DIR/bin/psql -d postgres -p 5434 -Atc "SELECT val FROM verify_replication WHERE id=1;" 2>/dev/null)
@@ -121,7 +122,7 @@ verify_logical_replication() {
121122
local delay=5
122123
local all_tables_replicated=true
123124

124-
for ((i=1; i<=TABLES; i++)); do
125+
for ((i=1; i<=$TABLES; i++)); do
125126
table_name="sbtest$i"
126127
echo "Checking table: $table_name"
127128

@@ -167,13 +168,13 @@ run_workload_during_conversion() {
167168
--table-size=1000 \
168169
--time=30 \
169170
run > /tmp/workload.log 2>&1 &
170-
WORKLOAD_PID=$!
171171
}
172172

173173
run_pg_createsubscriber() {
174174
echo "Running pg_createsubscriber..."
175175

176176
stop_server $REPLICA_DATA
177+
rm -f $REPLICA_DATA/postmaster.pid
177178

178179
$INSTALL_DIR/bin/pg_createsubscriber \
179180
-d postgres \
@@ -183,6 +184,7 @@ run_pg_createsubscriber() {
183184
--publisher-server="host=127.0.0.1 port=5433 dbname=postgres user=$(whoami)" \
184185
--publication=mypub \
185186
--subscription=mysub \
187+
--socketdir=/tmp \
186188
--verbose
187189

188190
start_server $REPLICA_DATA
@@ -206,11 +208,13 @@ echo "5=>Verifying Streaming Replication"
206208
verify_streaming_replication
207209

208210
echo "6=>Run workload in parallel on Primary Server"
209-
rotate_server_key &
211+
#rotate_server_key &
210212
run_workload_during_conversion
211213

212214
echo "7=>Convert physical replica into logical replica"
213-
run_pg_createsubscriber
215+
#run_pg_createsubscriber
216+
stop_server $REPLICA_DATA
217+
start_server $REPLICA_DATA
214218

215219
#echo "8=>Verifying Logical Replication"
216220
#verify_logical_replication

‎postgresql/tests/PG-1813.sh renamed to ‎postgresql/bugs/PG-1809/PG-1809.sh

Lines changed: 38 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ PG_BASEBACKUP="$INSTALL_DIR/bin/pg_basebackup"
1212
PSQL="$INSTALL_DIR/bin/psql"
1313
SYSBENCH="/usr/bin/sysbench"
1414

15-
PORT_PRIMARY=5432
16-
PORT_REPLICA=5433
15+
PORT_PRIMARY=5433
16+
PORT_REPLICA=5434
1717
REPL_USER=repl
1818
REPL_PASS=replica
1919
DB_NAME=postgres
@@ -22,22 +22,18 @@ DB_USER=$(whoami)
2222
# Clean slate
2323
pkill -9 postgres
2424
rm -rf "$PRIMARY_DATA" "$REPLICA_DATA"
25-
rm -rf /tmp/primary_keyfile
2625

2726
# Step 1: Init primary
2827
$INSTALL_DIR/bin/initdb -D "$PRIMARY_DATA"
2928

3029
# Configure primary
3130
cat >> "$PRIMARY_DATA/postgresql.conf" <<EOF
3231
port = $PORT_PRIMARY
33-
shared_preload_libraries = 'pg_tde'
34-
default_table_access_method = 'tde_heap'
3532
wal_level = replica
36-
wal_compression = on
3733
wal_log_hints = on
38-
wal_keep_size = 512MB
34+
wal_keep_size = 1024MB
3935
max_replication_slots = 2
40-
max_wal_senders = 2
36+
max_wal_senders = 10
4137
listen_addresses = 'localhost'
4238
logging_collector = on
4339
log_directory = 'log'
@@ -51,92 +47,87 @@ echo "=> Step 1: Start primary"
5147
echo "#########################"
5248
$PG_CTL -D "$PRIMARY_DATA" -o "-p $PORT_PRIMARY" -l "$PRIMARY_LOGFILE" start
5349
sleep 3
54-
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE EXTENSION pg_tde;"
55-
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_add_global_key_provider_file('file_provider','/tmp/primary_keyfile');"
56-
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_create_key_using_global_key_provider('key1','file_provider');"
57-
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_set_default_key_using_global_key_provider('key1','file_provider');"
58-
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "ALTER SYSTEM SET pg_tde.wal_encrypt=ON;"
59-
# Restart primary
60-
$PG_CTL -D "$PRIMARY_DATA" -o "-p $PORT_PRIMARY" -l "$PRIMARY_LOGFILE" restart
61-
sleep 3
50+
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE TABLE t1(a int, b varchar(20));"
6251

6352
# Create replication user
6453
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE ROLE $REPL_USER WITH LOGIN REPLICATION SUPERUSER PASSWORD '$REPL_PASS';"
6554

66-
# Create tables
67-
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE TABLE t1(id INT, name TEXT);"
68-
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES(101,'First Record Before BaseBackup');"
69-
7055
echo "=> Step 2: Take base backup for replica"
7156
echo "#######################################"
7257
$PG_BASEBACKUP -D "$REPLICA_DATA" -X stream -R -h localhost -p $PORT_PRIMARY -U $REPL_USER
7358

7459
# Configure replica
7560
cat > "$REPLICA_DATA/postgresql.conf" <<EOF
7661
port = $PORT_REPLICA
77-
shared_preload_libraries = 'pg_tde'
78-
default_table_access_method = 'tde_heap'
7962
hot_standby = on
8063
logging_collector = on
8164
log_directory = 'log'
8265
log_filename = 'replica.log'
83-
wal_level = replica
84-
wal_compression = on
85-
wal_keep_size= 512MB
86-
max_wal_senders = 2
66+
wal_keep_size=1024MB
67+
max_wal_senders = 10
8768
EOF
8869

8970
echo "=>Step 3: Start replica"
9071
echo "########################"
9172
$PG_CTL -D "$REPLICA_DATA" -o "-p $PORT_REPLICA" -l "$REPLICA_LOGFILE" start
92-
sleep 5
93-
$PSQL -p $PORT_REPLICA -d $DB_NAME -c "ALTER SYSTEM SET pg_tde.wal_encrypt=ON;"
94-
# Restart replica
95-
$PG_CTL -D "$REPLICA_DATA" -o "-p $PORT_REPLICA" -l "$REPLICA_LOGFILE" restart
96-
sleep 5
73+
sleep 2
74+
75+
echo "=> Check if replication is working"
76+
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES(1,'mohit');"
77+
sleep 2
9778

98-
echo "=>Step 4: Generate some WAL on primary"
99-
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES (102, 'Second Record After Setting Streaming Replication');"
79+
echo "=> Verify if rows get replicated"
80+
$PSQL -p $PORT_REPLICA -d $DB_NAME -c "SELECT * FROM t1;"
10081

82+
echo "=>Step 4: Stop primary (simulate crash)"
83+
echo "#######################################"
84+
echo "Simulating primary crash..."
10185
$PG_CTL -D "$PRIMARY_DATA" -m immediate stop
102-
sleep 3
10386

10487
echo "=> Step 5: Promote replica"
10588
echo "##########################"
106-
# Remove auto conf file which was created by pg_basebackup
107-
rm -f $REPLICA_DATA/postgresql.auto.conf
10889
$PG_CTL -D "$REPLICA_DATA" promote
90+
sleep 5
10991

110-
echo "=>Step 6: Use sysbench to generate WAL on promoted replica"
92+
echo "=>Step 6: Create table to generate WAL on promoted replica"
11193
echo "##########################################################"
112-
echo "Generating WAL with sysbench..."
113-
$PSQL -p $PORT_REPLICA -d $DB_NAME -c "INSERT INTO t1 VALUES(103,'Third Record Replicated Via PgRewind');"
94+
echo "Generating WAL..."
95+
$PSQL -p $PORT_REPLICA -d $DB_NAME -c "CREATE TABLE t2(a int, b varchar(20));"
11496

11597
# Backing up the Original config file
11698
cp $PRIMARY_DATA/postgresql.conf /tmp/postgresql_bk.conf
117-
echo "=> Step 8: Rewind old primary"
99+
echo "=> Step 7: Rewind old primary"
118100
echo "############################"
119101
echo "Rewinding old primary..."
120102
$PG_REWIND --target-pgdata="$PRIMARY_DATA" \
121103
--source-server="host=localhost port=$PORT_REPLICA user=$REPL_USER dbname=$DB_NAME"
122104

123105
# Restoring the Original config file
124-
cp /tmp/postgresql_bk.conf $PRIMARY_DATA/postgresql.conf
106+
mv /tmp/postgresql_bk.conf $PRIMARY_DATA/postgresql.conf
125107

126-
echo "=> Step 9: Configure old primary as standby"
108+
echo "=> Step 8: Configure old primary as standby"
127109
echo "###########################################"
128110
touch $PRIMARY_DATA/standby.signal
129111
cat >> "$PRIMARY_DATA/postgresql.conf" <<EOF
130112
primary_conninfo = 'host=localhost port=$PORT_REPLICA user=$REPL_USER password=$REPL_PASS'
131113
EOF
132114

133-
echo "=> Step 10: Start rewound primary"
115+
echo "=> Step 9: Start rewound primary"
134116
echo "################################"
135117
$PG_CTL -D "$PRIMARY_DATA" -o "-p $PORT_PRIMARY" -l "$PRIMARY_LOGFILE" start
136118
sleep 5
137119

138-
# Validate data on both servers
139-
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT * FROM t1"
140-
$PSQL -p $PORT_REPLICA -d $DB_NAME -c "SELECT * FROM t1"
120+
echo "=> Check if replication is working"
121+
$PSQL -p $PORT_REPLICA -d $DB_NAME -c "INSERT INTO t2 VALUES(2,'mohit');"
122+
sleep 2
123+
124+
echo "=> Verify if rows get replicated"
125+
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT * FROM t2;"
126+
127+
128+
# Done
129+
echo -e "\n✅ Check status of Old Primary"
130+
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT * FROM pg_stat_wal_receiver;"
141131

142-
tail -10f $PRIMARY_DATA/log/primary.log
132+
# Tail logs
133+
tail -n 10 "$PRIMARY_DATA/log/primary.log"

‎postgresql/bugs/PG-1811/PG-1811.sh

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ PG_REWIND="$INSTALL_DIR/bin/pg_rewind"
1111
PG_BASEBACKUP="$INSTALL_DIR/bin/pg_basebackup"
1212
PSQL="$INSTALL_DIR/bin/psql"
1313
SYSBENCH="/usr/bin/sysbench"
14+
SYSBENCH_TABLES=1000
15+
SYSBENCH_RECORDS=1000
16+
THREADS=10
17+
PGCTLTIMEOUT=600
1418

1519
PORT_PRIMARY=5433
1620
PORT_REPLICA=5434
@@ -34,7 +38,7 @@ default_table_access_method = 'tde_heap'
3438
port = $PORT_PRIMARY
3539
wal_level = replica
3640
wal_log_hints = on
37-
wal_keep_size = 1024MB
41+
wal_keep_size = 2048MB
3842
max_replication_slots = 2
3943
max_wal_senders = 10
4044
listen_addresses = 'localhost'
@@ -55,10 +59,10 @@ $PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_add_global_key_provider_fil
5559
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_create_key_using_global_key_provider('key1','file_provider');"
5660
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_set_server_key_using_global_key_provider('key1','file_provider');"
5761
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT pg_tde_set_key_using_global_key_provider('key1','file_provider');"
58-
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "ALTER SYSTEM SET pg_tde.wal_encrypt=ON;"
62+
#$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "ALTER SYSTEM SET pg_tde.wal_encrypt=ON;"
5963
# Restart primary
60-
$PG_CTL -D "$PRIMARY_DATA" -o "-p $PORT_PRIMARY" -l "$PRIMARY_LOGFILE" restart
61-
sleep 3
64+
#$PG_CTL -D "$PRIMARY_DATA" -o "-p $PORT_PRIMARY" -l "$PRIMARY_LOGFILE" restart
65+
#sleep 3
6266

6367
# Create replication user
6468
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE ROLE $REPL_USER WITH LOGIN REPLICATION SUPERUSER PASSWORD '$REPL_PASS';"
@@ -84,10 +88,10 @@ echo "=>Step 3: Start replica"
8488
echo "########################"
8589
$PG_CTL -D "$REPLICA_DATA" -o "-p $PORT_REPLICA" -l "$REPLICA_LOGFILE" start
8690
sleep 5
87-
$PSQL -p $PORT_REPLICA -d $DB_NAME -c "ALTER SYSTEM SET pg_tde.wal_encrypt=ON;"
91+
#$PSQL -p $PORT_REPLICA -d $DB_NAME -c "ALTER SYSTEM SET pg_tde.wal_encrypt=ON;"
8892
# Restart replica
89-
$PG_CTL -D "$REPLICA_DATA" -o "-p $PORT_REPLICA" -l "$REPLICA_LOGFILE" restart
90-
sleep 5
93+
#$PG_CTL -D "$REPLICA_DATA" -o "-p $PORT_REPLICA" -l "$REPLICA_LOGFILE" restart
94+
#sleep 5
9195

9296
echo "=>Step 3: Stop primary (simulate crash)"
9397
echo "#######################################"
@@ -105,12 +109,12 @@ echo "##########################################################"
105109
echo "Generating WAL with sysbench..."
106110
$SYSBENCH --db-driver=pgsql --pgsql-host=127.0.0.1 --pgsql-port=$PORT_REPLICA \
107111
--pgsql-user=$DB_USER --pgsql-db=$DB_NAME \
108-
--threads=10 --tables=1000 --table-size=1000 \
112+
--threads=$THREADS --tables=$SYSBENCH_TABLES --table-size=$SYSBENCH_RECORDS \
109113
/usr/share/sysbench/oltp_write_only.lua prepare
110114

111115
$SYSBENCH --db-driver=pgsql --pgsql-host=127.0.0.1 --pgsql-port=$PORT_REPLICA \
112116
--pgsql-user=$DB_USER --pgsql-db=$DB_NAME \
113-
--threads=10 --tables=1000 --table-size=1000 --time=300 --report-interval=5 \
117+
--threads=$THREADS --tables=$SYSBENCH_TABLES --table-size=$SYSBENCH_RECORDS --time=200 --report-interval=5 \
114118
/usr/share/sysbench/oltp_write_only.lua run
115119

116120
# Backing up the Original config file

‎postgresql/bugs/PG-1813/PG-1813.sh

100644100755
Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ $PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE ROLE $REPL_USER WITH LOGIN REPLICA
6565

6666
# Create tables
6767
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "CREATE TABLE t1(id INT, name TEXT);"
68-
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES(101,'First Record');"
68+
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES(101,'First Record Before BaseBackup');"
6969

7070
echo "=> Step 2: Take base backup for replica"
7171
echo "#######################################"
@@ -96,7 +96,7 @@ $PG_CTL -D "$REPLICA_DATA" -o "-p $PORT_REPLICA" -l "$REPLICA_LOGFILE" restart
9696
sleep 5
9797

9898
echo "=>Step 4: Generate some WAL on primary"
99-
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES (102, 'Second Record');"
99+
$PSQL -p $PORT_PRIMARY -d $DB_NAME -c "INSERT INTO t1 VALUES (102, 'Second Record After Setting Streaming Replication');"
100100

101101
$PG_CTL -D "$PRIMARY_DATA" -m immediate stop
102102
sleep 3
@@ -110,7 +110,7 @@ $PG_CTL -D "$REPLICA_DATA" promote
110110
echo "=>Step 6: Use sysbench to generate WAL on promoted replica"
111111
echo "##########################################################"
112112
echo "Generating WAL with sysbench..."
113-
$PSQL -p $PORT_REPLICA -d $DB_NAME -c "INSERT INTO t1 VALUES(103,'Third Record');"
113+
$PSQL -p $PORT_REPLICA -d $DB_NAME -c "INSERT INTO t1 VALUES(103,'Third Record Replicated Via PgRewind');"
114114

115115
# Backing up the Original config file
116116
cp $PRIMARY_DATA/postgresql.conf /tmp/postgresql_bk.conf
@@ -140,4 +140,3 @@ $PSQL -p $PORT_PRIMARY -d $DB_NAME -c "SELECT * FROM t1"
140140
$PSQL -p $PORT_REPLICA -d $DB_NAME -c "SELECT * FROM t1"
141141

142142
tail -10f $PRIMARY_DATA/log/primary.log
143-

0 commit comments

Comments
 (0)