From 19288ae777c6dd305206c1102965202deca7351c Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Wed, 13 Sep 2023 11:44:04 -0700 Subject: [PATCH 1/2] docs draft1 --- docs/aws-gpc-storage-walkthrough.md | 167 ++++++++++++++++++ helm-chart/eoapi/templates/db/configmap.yaml | 2 + helm-chart/eoapi/templates/db/deployment.yaml | 2 + helm-chart/eoapi/templates/db/pvc.yaml | 4 +- helm-chart/eoapi/templates/db/service.yaml | 2 + 5 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 docs/aws-gpc-storage-walkthrough.md diff --git a/docs/aws-gpc-storage-walkthrough.md b/docs/aws-gpc-storage-walkthrough.md new file mode 100644 index 0000000..5ead457 --- /dev/null +++ b/docs/aws-gpc-storage-walkthrough.md @@ -0,0 +1,167 @@ +# Production Storage + +Most folks will want to use a cloud provider's PasS database service for production situations. Below +we walkthrough the chnages to make that doable + +--- + +## AWS RDS + +Step 1: First, we'll need to provision an RDS or Cloud SQL instance (see [GCP](#gcp-cloud-sql) below) + +> ⓘ The Terraform below is just for edification and examples. +> It's not meant to be exhaustive something you execute from this repo + +```terraform +terraform { + required_version = "1.3.9" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.0" + } + } +} + +resource "aws_db_subnet_group" "db" { + name = "tf-${var.project_name}-${var.env}-subnet-group" + subnet_ids = ${var.private_subnet_ids_array} + tags = { + Name = "tf-${var.project_name}-subnet-group" + } +} + +# NOTE: the below params are just examples of modifications +# they have nothing to do with what is a recommended default +# b/c that depends largely on your data and other variables +resource "aws_db_parameter_group" "default" { + name = "tf-${var.project_name}-${var.env}-postgres14-param-group" + family = "postgres14" + + parameter { + name = "work_mem" + value = "8192" + } + + parameter { + name = "max_connections" + value = "475" + apply_method = "pending-reboot" + } + + parameter { + name = "shared_buffers" + value = "4032428" + apply_method = "pending-reboot" + } + + parameter { + name = "seq_page_cost" + value = "1" + } + + parameter { + name = "random_page_cost" + value = "1.2" + } +} + +resource "aws_db_instance" "db" { + db_name = "eoapi-prod-db" + identifier = "${var.project_name}-${var.env}" + engine = "postgres" + engine_version = "14.7" + // https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_CreateDBInstance.html + allocated_storage = 100 + max_allocated_storage = 500 + storage_type = "gp2" + instance_class = "db.r5.large" + db_subnet_group_name = aws_db_subnet_group.db.name + vpc_security_group_ids = security_group_ids_array + skip_final_snapshot = true + apply_immediately = true + backup_retention_period = 7 + username = "postgres" + password = var.db_password + allow_major_version_upgrade = true + parameter_group_name = aws_db_parameter_group.default.name +} + +output "db_hostname" { + value = aws_db_instance.db.endpoint +} +``` + +## GCP Cloud SQL + +```terraform +terraform { + required_version = "1.3.9" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 3.5.0" + } + } +} + +resource "google_sql_database_instance" "default" { + name = "eoapi-prod-db-instance" + database_version = "POSTGRES_14" + region = "us-central1" + + settings { + tier = "db-n1-standard-2" + + ip_configuration { + ipv4_enabled = true + private_network = ${var.vpc_network_self_link} + + authorized_networks { + value = "0.0.0.0/0" # Caution: This opens to all IPs + name = "all-ips" + } + } + } +} + +resource "google_sql_database" "default" { + name = "eoapi-prod-db" + instance = google_sql_database_instance.default.name + collation = "en_US.UTF8" +} + +resource "google_sql_user" "users" { + name = "postgres" + instance = google_sql_database_instance.default.name + password = "${var.db_password} +} + +output "instance_address" { + value = google_sql_database_instance.default.ip_address[0].ip_address +} +``` + +## Write eoapi-k8s config.yaml + +Step 2: Next we just need to develop some `config.yaml` overrides that `helm install` will use with our new host, port, username etc + +```bash + $ cat config.yaml + db: + environment: "rds" + settings: + secrets: + POSTGRES_DB: "postgis" + POSTGRES_USER: "" + POSTGRES_PASSWORD: "" + POSTGRES_PORT: "5432" + POSTGRES_HOST: "" + POSTGRES_HOST_READER: "" + POSTGRES_HOST_WRITER: "" + # default connect: https://www.postgresql.org/docs/current/libpq-envars.html + PGDATA: "/var/lib/postgresql/data/pgdata" + PGUSER: "" + PGPASSWORD: "" + PGDATABASE: "postgis" +``` \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/configmap.yaml b/helm-chart/eoapi/templates/db/configmap.yaml index 9ec2b2a..716de74 100644 --- a/helm-chart/eoapi/templates/db/configmap.yaml +++ b/helm-chart/eoapi/templates/db/configmap.yaml @@ -1,3 +1,4 @@ +{{- if (eq .Values.db.environment "k8s") }} --- apiVersion: v1 kind: ConfigMap @@ -32,3 +33,4 @@ data: psql $DSN -f /opt/initdb/sql-data/initdb.sql # run it forever like a docker process should tail -f /dev/null +{{- end }} \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/deployment.yaml b/helm-chart/eoapi/templates/db/deployment.yaml index 7c58b06..ca2d4a8 100644 --- a/helm-chart/eoapi/templates/db/deployment.yaml +++ b/helm-chart/eoapi/templates/db/deployment.yaml @@ -1,3 +1,4 @@ +{{- if (eq .Values.db.environment "k8s") }} --- apiVersion: apps/v1 kind: Deployment @@ -71,3 +72,4 @@ spec: - name: initdb-sh-volume-{{ $.Release.Name }} configMap: name: initdb-sh-config-{{ $.Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/pvc.yaml b/helm-chart/eoapi/templates/db/pvc.yaml index a28af8f..eaf48b0 100644 --- a/helm-chart/eoapi/templates/db/pvc.yaml +++ b/helm-chart/eoapi/templates/db/pvc.yaml @@ -1,3 +1,4 @@ +{{- if (eq .Values.db.environment "k8s") }} --- apiVersion: v1 kind: PersistentVolumeClaim @@ -8,4 +9,5 @@ spec: - ReadWriteOnce resources: requests: - storage: {{ .Values.db.settings.resources.requests.storage }} \ No newline at end of file + storage: {{ .Values.db.settings.resources.requests.storage }} +{{- end }} \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/service.yaml b/helm-chart/eoapi/templates/db/service.yaml index 2854eba..c270b86 100644 --- a/helm-chart/eoapi/templates/db/service.yaml +++ b/helm-chart/eoapi/templates/db/service.yaml @@ -1,3 +1,4 @@ +{{- if (eq .Values.db.environment "k8s") }} --- apiVersion: v1 kind: Service @@ -12,3 +13,4 @@ spec: targetPort: 5432 selector: app: {{ include "eoapi.pgstacHostName" . | nindent 10 }} +{{- end }} From 942437a7be62f04ae7cc422f220b41df400bbcf9 Mon Sep 17 00:00:00 2001 From: ranchodeluxe Date: Fri, 15 Sep 2023 12:46:53 -0700 Subject: [PATCH 2/2] moar --- docs/aws-gpc-storage-walkthrough.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/aws-gpc-storage-walkthrough.md b/docs/aws-gpc-storage-walkthrough.md index 5ead457..82c0a8d 100644 --- a/docs/aws-gpc-storage-walkthrough.md +++ b/docs/aws-gpc-storage-walkthrough.md @@ -1,7 +1,7 @@ # Production Storage Most folks will want to use a cloud provider's PasS database service for production situations. Below -we walkthrough the chnages to make that doable +we walkthrough the changes to make that doable ---