Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Correct configuration for target_cluster_endpoint and target_cluster_region within the SimpleReplayUtility #607

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+22 −15
Open

Correct configuration for target_cluster_endpoint and target_cluster_region within the SimpleReplayUtility #607

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f311171
Fix misconfiguration on target endpoint and region for SimpleReplay
Feb 22, 2022
5a433c1
Merge branch 'master' of github.com:ctcudd/amazon-redshift-utils
Feb 22, 2022
1603f2c
Remove duplicate logic for parsing cluster region/host/port/database …
Feb 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions src/SimpleReplay/cloudformation/ra3-migration-replay.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,14 @@ Parameters:
Description: AMI for the Amazon Linux 2 based EC2 instance
Type: "AWS::SSM::Parameter::Value<String>"
Default: "/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2"

GitOrigin:
Description: Git repository to source SimpleReplay utility from
Type: String
Default: "https://github.com/awslabs/amazon-redshift-utils.git"
GitBranch:
Description: Git branch to checkout
Type: String
Default: "master"
Metadata:
AWS::CloudFormation::Interface:
ParameterGroups:
Expand Down Expand Up @@ -1159,7 +1166,7 @@ Resources:
echo "region = ${AWS::Region}" >> /root/.aws/config
mkdir /amazonutils
cd /amazonutils
git clone https://github.com/awslabs/amazon-redshift-utils.git
git clone ${GitOrigin} -b ${GitBranch}
pip3 install -r /amazonutils/amazon-redshift-utils/src/SimpleReplay/requirements.txt
#
# configure replay metadata
Expand Down Expand Up @@ -1230,7 +1237,7 @@ Resources:
echo "region = ${AWS::Region}" >> /root/.aws/config
mkdir /amazonutils
cd /amazonutils
git clone https://github.com/awslabs/amazon-redshift-utils.git
git clone ${GitOrigin} -b ${GitBranch}
pip3 install -r /amazonutils/amazon-redshift-utils/src/SimpleReplay/requirements.txt
cd /amazonutils/amazon-redshift-utils/src/SimpleReplay
mkdir -p cloudformation
Expand Down
2 changes: 1 addition & 1 deletion src/SimpleReplay/cloudformation/run_replay.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ cd /amazonutils/amazon-redshift-utils/src/SimpleReplay
mkdir -p $bucket_keyprefix
aws s3 cp s3://$replay_bucket/config/replay.yaml ./$bucket_keyprefix/replay_target.yaml
sed -i "s#workload_location: \"\"#workload_location: \"s3://$extract_bucket/$bucket_keyprefix/extract/$extract_output\"#g" ./$bucket_keyprefix/replay_target.yaml
sed -i "s#target_cluster_endpoint: \"\"#target_cluster_endpoint: \"$cluster_endpoint\"#g" ./$bucket_keyprefix/replay_target.yaml
sed -i "s#target_cluster_endpoint: \"host:port/database\"#target_cluster_endpoint: \"$cluster_endpoint\"#g" ./$bucket_keyprefix/replay_target.yaml
#
cp -f ./$bucket_keyprefix/replay_target.yaml ./$bucket_keyprefix/replay_replica.yaml
sed -i "s#replay_output: \"\"#replay_output: \"s3://$replay_bucket/$bucket_keyprefix/replay/replay_output_target\"#g" ./$bucket_keyprefix/replay_target.yaml
Expand Down
22 changes: 11 additions & 11 deletions src/SimpleReplay/replay.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
g_config = {}

g_replay_timestamp = None

g_cluster = {}

class ConnectionLog:
def __init__(
Expand Down Expand Up @@ -1317,14 +1317,12 @@ def get_connection_credentials(username, database=None, max_attempts=10, skip_ca
return record['target_cluster_urls']
del g_credentials_cache[username]

cluster_endpoint = g_config["target_cluster_endpoint"]
odbc_driver = g_config["odbc_driver"]

cluster_endpoint_split = cluster_endpoint.split(".")
cluster_id = cluster_endpoint_split[0]
cluster_host = cluster_endpoint.split(":")[0]
cluster_port = cluster_endpoint_split[5].split("/")[0][4:]
cluster_database = database or cluster_endpoint_split[5].split("/")[1]
cluster_id = g_cluster.get('id')
cluster_host = g_cluster.get('host')
cluster_port = g_cluster.get('port')
cluster_database = database or g_cluster.get('database')

additional_args = {}
if os.environ.get('ENDPOINT_URL'):
Expand All @@ -1335,7 +1333,7 @@ def get_connection_credentials(username, database=None, max_attempts=10, skip_ca
'verify': False}

response = None
rs_client = client("redshift", region_name=g_config.get("target_cluster_region", None), **additional_args)
rs_client = client("redshift", region_name=g_cluster.get("region"), **additional_args)
for attempt in range(1, max_attempts + 1):
try:
response = rs_client.get_cluster_credentials(
Expand Down Expand Up @@ -1612,17 +1610,19 @@ def main():

# print the version
log_version()
cluster = cluster_dict(g_config["target_cluster_endpoint"])

global g_cluster
g_cluster = cluster_dict(g_config["target_cluster_endpoint"])
# generate replay id/hash
global g_replay_timestamp
g_replay_timestamp = datetime.datetime.now(tz=datetime.timezone.utc)
logger.info(f"Replay start time: {g_replay_timestamp}")

id_hash = hashlib.sha1(g_replay_timestamp.isoformat().encode("UTF-8")).hexdigest()[:5]
if g_config.get("tag", "") != "":
replay_id = f'{g_replay_timestamp.isoformat()}_{cluster.get("id")}_{g_config["tag"]}_{id_hash}'
replay_id = f'{g_replay_timestamp.isoformat()}_{g_cluster.get("id")}_{g_config["tag"]}_{id_hash}'
else:
replay_id = f'{g_replay_timestamp.isoformat()}_{cluster.get("id")}_{id_hash}'
replay_id = f'{g_replay_timestamp.isoformat()}_{g_cluster.get("id")}_{id_hash}'

manager = SyncManager()

Expand Down