From 9d01c3889c77347521213105701cc3bd8644236e Mon Sep 17 00:00:00 2001 From: Cliff Casey Date: Wed, 15 May 2024 08:58:51 -0400 Subject: [PATCH 1/8] two noisy example --- dev-resources/tla-demo/docker-compose.yml | 156 ++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 dev-resources/tla-demo/docker-compose.yml diff --git a/dev-resources/tla-demo/docker-compose.yml b/dev-resources/tla-demo/docker-compose.yml new file mode 100644 index 000000000..64a0e2453 --- /dev/null +++ b/dev-resources/tla-demo/docker-compose.yml @@ -0,0 +1,156 @@ +# Example Architecture with Two Noisy LRS' and one Transactional LRS + +version: "3.9" + +volumes: + nsy_lrs_1_db_data: + nsy_lrs_2_db_data: + txn_lrs_db_data: + lrspipe_storage: + +services: + # Noisy LRS 1 + # - Noisy LRS 1: DB + nsy_lrs_1_db: + image: postgres:14 + volumes: + - nsy_lrs_1_db_data:/var/lib/postgresql/data + environment: + POSTGRES_USER: lrsql_user + POSTGRES_PASSWORD: lrsql_password + POSTGRES_DB: lrsql_db + # - Noisy LRS 1: App + nsy_lrs_1: + image: yetanalytics/lrsql:v0.7.13 + command: + - /lrsql/bin/run_postgres.sh + ports: + - "8081:8080" + depends_on: + - nsy_lrs_1_db + environment: + LRSQL_API_KEY_DEFAULT: my_key + LRSQL_API_SECRET_DEFAULT: my_secret + LRSQL_ADMIN_USER_DEFAULT: my_username + LRSQL_ADMIN_PASS_DEFAULT: my_password + LRSQL_DB_HOST: nsy_lrs_1_db + LRSQL_DB_NAME: lrsql_db + LRSQL_DB_USER: lrsql_user + LRSQL_DB_PASSWORD: lrsql_password + LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000 + LRSQL_ENABLE_REACTIONS: true + LRSQL_ALLOW_ALL_ORIGINS: true + restart: always + + # Noisy LRS 2 + # - Noisy LRS 2: DB + nsy_lrs_2_db: + image: postgres:14 + volumes: + - nsy_lrs_2_db_data:/var/lib/postgresql/data + environment: + POSTGRES_USER: lrsql_user + POSTGRES_PASSWORD: lrsql_password + POSTGRES_DB: lrsql_db + # - Noisy LRS 2: App + nsy_lrs_2: + image: yetanalytics/lrsql:v0.7.13 + command: + - /lrsql/bin/run_postgres.sh + ports: + - "8082:8080" + depends_on: + - nsy_lrs_2_db + environment: + LRSQL_API_KEY_DEFAULT: my_key + LRSQL_API_SECRET_DEFAULT: my_secret + LRSQL_ADMIN_USER_DEFAULT: my_username + LRSQL_ADMIN_PASS_DEFAULT: my_password + LRSQL_DB_HOST: nsy_lrs_2_db + LRSQL_DB_NAME: lrsql_db + LRSQL_DB_USER: lrsql_user + LRSQL_DB_PASSWORD: lrsql_password + LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000 + LRSQL_ENABLE_REACTIONS: true + LRSQL_ALLOW_ALL_ORIGINS: true + restart: always + + # Transactional LRS + # - Transactional LRS: DB + txn_lrs_db: + image: postgres:14 + volumes: + - txn_lrs_db_data:/var/lib/postgresql/data + environment: + POSTGRES_USER: lrsql_user + POSTGRES_PASSWORD: lrsql_password + POSTGRES_DB: lrsql_db + # - Transactional LRS: App + txn_lrs: + image: yetanalytics/lrsql:v0.7.13 + command: + - /lrsql/bin/run_postgres.sh + ports: + - "8090:8080" + depends_on: + - txn_lrs_db + environment: + LRSQL_API_KEY_DEFAULT: my_key + LRSQL_API_SECRET_DEFAULT: my_secret + LRSQL_ADMIN_USER_DEFAULT: my_username + LRSQL_ADMIN_PASS_DEFAULT: my_password + LRSQL_DB_HOST: txn_lrs_db + LRSQL_DB_NAME: lrsql_db + LRSQL_DB_USER: lrsql_user + LRSQL_DB_PASSWORD: lrsql_password + LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000 + LRSQL_ENABLE_REACTIONS: true + LRSQL_ALLOW_ALL_ORIGINS: true + restart: always + + # LRSPipe + # - LRSPipe: Shared Redis + redis: + image: redis:6-alpine + volumes: + - lrspipe_storage:/data + ports: + - "6379" + # - LRSPipe: Sync from Noisy LRS 1 to Transactional LRS + lrspipe1: + image: yetanalytics/xapipe:latest + depends_on: + - nsy_lrs_1 + - txn_lrs + - redis + command: | + -s redis + --job-id nsy_lrs_1_sync + -f + --redis-uri redis://redis:6379?db=nsy_lrs_1_sync + --source-url http://nsy_lrs_1:8080/xapi + --source-username my_key + --source-password my_secret + --target-url http://txn_lrs:8080/xapi + --target-username my_key + --target-password my_secret + restart: always + # - LRSPipe: Sync from Noisy LRS 2 to Transactional LRS + lrspipe2: + image: yetanalytics/xapipe:latest + depends_on: + - nsy_lrs_2 + - txn_lrs + - redis + command: | + -s redis + --job-id nsy_lrs_2_sync + -f + --redis-uri redis://redis:6379?db=nsy_lrs_2_sync + --source-url http://nsy_lrs_2:8080/xapi + --source-username my_key + --source-password my_secret + --target-url http://txn_lrs:8080/xapi + --target-username my_key + --target-password my_secret + restart: always From 32822558ad6d96f6a2c204aecb14af4201a7be96 Mon Sep 17 00:00:00 2001 From: Cliff Casey Date: Wed, 29 May 2024 10:58:58 -0400 Subject: [PATCH 2/8] expose transactional db port as 5431 --- dev-resources/tla-demo/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-resources/tla-demo/docker-compose.yml b/dev-resources/tla-demo/docker-compose.yml index 64a0e2453..6c07ed939 100644 --- a/dev-resources/tla-demo/docker-compose.yml +++ b/dev-resources/tla-demo/docker-compose.yml @@ -85,6 +85,8 @@ services: POSTGRES_USER: lrsql_user POSTGRES_PASSWORD: lrsql_password POSTGRES_DB: lrsql_db + ports: + - "5431:5432" # - Transactional LRS: App txn_lrs: image: yetanalytics/lrsql:v0.7.13 From c6726a376ce5685ac68a8d9533c90ee40f4b27f8 Mon Sep 17 00:00:00 2001 From: Cliff Casey Date: Tue, 24 Sep 2024 10:05:35 -0400 Subject: [PATCH 3/8] added third noisy --- dev-resources/tla-demo/docker-compose.yml | 59 +++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/dev-resources/tla-demo/docker-compose.yml b/dev-resources/tla-demo/docker-compose.yml index 6c07ed939..08aaa43f4 100644 --- a/dev-resources/tla-demo/docker-compose.yml +++ b/dev-resources/tla-demo/docker-compose.yml @@ -5,6 +5,7 @@ version: "3.9" volumes: nsy_lrs_1_db_data: nsy_lrs_2_db_data: + nsy_lrs_3_db_data: txn_lrs_db_data: lrspipe_storage: @@ -21,7 +22,7 @@ services: POSTGRES_DB: lrsql_db # - Noisy LRS 1: App nsy_lrs_1: - image: yetanalytics/lrsql:v0.7.13 + image: yetanalytics/lrsql:v0.7.20 command: - /lrsql/bin/run_postgres.sh ports: @@ -54,7 +55,7 @@ services: POSTGRES_DB: lrsql_db # - Noisy LRS 2: App nsy_lrs_2: - image: yetanalytics/lrsql:v0.7.13 + image: yetanalytics/lrsql:v0.7.20 command: - /lrsql/bin/run_postgres.sh ports: @@ -75,6 +76,39 @@ services: LRSQL_ALLOW_ALL_ORIGINS: true restart: always + # Noisy LRS 3 + # - Noisy LRS 3: DB + nsy_lrs_3_db: + image: postgres:14 + volumes: + - nsy_lrs_3_db_data:/var/lib/postgresql/data + environment: + POSTGRES_USER: lrsql_user + POSTGRES_PASSWORD: lrsql_password + POSTGRES_DB: lrsql_db + # - Noisy LRS 3: App + nsy_lrs_3: + image: yetanalytics/lrsql:v0.7.20 + command: + - /lrsql/bin/run_postgres.sh + ports: + - "8083:8080" + depends_on: + - nsy_lrs_3_db + environment: + LRSQL_API_KEY_DEFAULT: my_key + LRSQL_API_SECRET_DEFAULT: my_secret + LRSQL_ADMIN_USER_DEFAULT: my_username + LRSQL_ADMIN_PASS_DEFAULT: my_password + LRSQL_DB_HOST: nsy_lrs_3_db + LRSQL_DB_NAME: lrsql_db + LRSQL_DB_USER: lrsql_user + LRSQL_DB_PASSWORD: lrsql_password + LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000 + LRSQL_ENABLE_REACTIONS: true + LRSQL_ALLOW_ALL_ORIGINS: true + restart: always + # Transactional LRS # - Transactional LRS: DB txn_lrs_db: @@ -89,7 +123,7 @@ services: - "5431:5432" # - Transactional LRS: App txn_lrs: - image: yetanalytics/lrsql:v0.7.13 + image: yetanalytics/lrsql:v0.7.20 command: - /lrsql/bin/run_postgres.sh ports: @@ -156,3 +190,22 @@ services: --target-username my_key --target-password my_secret restart: always + # - LRSPipe: Sync from Noisy LRS 3 to Transactional LRS + lrspipe3: + image: yetanalytics/xapipe:latest + depends_on: + - nsy_lrs_3 + - txn_lrs + - redis + command: | + -s redis + --job-id nsy_lrs_3_sync + -f + --redis-uri redis://redis:6379?db=nsy_lrs_3_sync + --source-url http://nsy_lrs_3:8080/xapi + --source-username my_key + --source-password my_secret + --target-url http://txn_lrs:8080/xapi + --target-username my_key + --target-password my_secret + restart: always From 32b36b9cdf8fae74a068d6104d0d4608d26ceaf4 Mon Sep 17 00:00:00 2001 From: Cliff Casey Date: Tue, 24 Sep 2024 15:07:57 -0400 Subject: [PATCH 4/8] additional demo explanations --- doc/other_demos.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 doc/other_demos.md diff --git a/doc/other_demos.md b/doc/other_demos.md new file mode 100644 index 000000000..620b26df1 --- /dev/null +++ b/doc/other_demos.md @@ -0,0 +1,32 @@ +[<- Back to Index](index.md) + +# Assorted Deployment Configuration Examples + +## Load Balanced LRS Demo + +This demo illustrates how SQL LRS can be configured with multiple load balanced application servers with a single PostgreSQL database. The important configuration variable to pay attention to for multiple nodes in a single cluster is `LRSQL_JWT_COMMON_SECRET`, which allows the servers to share JWTs. Alternatively you may be able to implement session-sticky server rotation at the load balancer level, depending on your load balancer. + +### Run the Docker Stack + + cd dev-resources/load_balanced + docker compose up + +## Proxied LRS Demo + +This demo illustrates how SQL LRS must be configured if you are using a proxy (like nginx) to serve SQL LRS on a custom path. This is useful, for instance, for when you cannot use a dedicated domain/subdomain and need to serve SQL LRS from a path like `https://www.yetanalytics.com/my-lrs/...`. The important configuration variable to note for this situation is `LRSQL_PROXY_PATH` which tells the frontend to look for the server at that path. *NOTE: This variable does not actually move the location of SQL LRS endpoints, that must be done with a proxy, instead it just makes the components aware that that is happening*. + +### Run the Docker Stack + + cd dev-resources/proxied_example + docker compose up + +## TLA Demo + +This demo illustrates a configuration similar to the Total Learning Architecture, wherein multiple Noisy LRS instances are feeding a single Transactional LRS. In this demo, three PostgreSQL-backed LRSs will be launched, three LRS Pipe processes will consume their data, and a Transactional LRS will receive the aggregation of that data. + +### Run the Docker Stack + + cd dev-resources/tla-demo + docker compose up + +[<- Back to Index](index.md) From bb65b23ac1d27aa115772e9509057bca2cc55ba8 Mon Sep 17 00:00:00 2001 From: Cliff Casey Date: Tue, 24 Sep 2024 15:10:41 -0400 Subject: [PATCH 5/8] demo links --- README.md | 1 + doc/index.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index d01569c67..ad5286488 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ For releases and release notes, see the [Releases](https://github.com/yetanalyti ### Demos - [Visualization with Apache Superset](doc/superset.md) +- [Additional Configuration Demos](doc/other_demos.md) ## Contribution diff --git a/doc/index.md b/doc/index.md index 77a3bc0b1..5fb3c0293 100644 --- a/doc/index.md +++ b/doc/index.md @@ -35,6 +35,7 @@ ### Demos - [Visualization with Apache Superset](superset.md) +- [Additional Configuration Demos](doc/other_demos.md) ### Releases From 71727827d0029f0b1c20094501650ce0843837e8 Mon Sep 17 00:00:00 2001 From: Cliff Casey Date: Tue, 24 Sep 2024 15:11:42 -0400 Subject: [PATCH 6/8] a word --- doc/other_demos.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/other_demos.md b/doc/other_demos.md index 620b26df1..88e2d7046 100644 --- a/doc/other_demos.md +++ b/doc/other_demos.md @@ -1,6 +1,6 @@ [<- Back to Index](index.md) -# Assorted Deployment Configuration Examples +# Additional Deployment Configuration Examples ## Load Balanced LRS Demo From 3a21817ba51eadae5d8b95b2ace4a86009dea6b0 Mon Sep 17 00:00:00 2001 From: Cliff Casey Date: Tue, 24 Sep 2024 17:18:15 -0400 Subject: [PATCH 7/8] health checks --- dev-resources/tla-demo/docker-compose.yml | 47 ++++++++++++++++++----- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/dev-resources/tla-demo/docker-compose.yml b/dev-resources/tla-demo/docker-compose.yml index 08aaa43f4..b69f8a2ba 100644 --- a/dev-resources/tla-demo/docker-compose.yml +++ b/dev-resources/tla-demo/docker-compose.yml @@ -29,6 +29,11 @@ services: - "8081:8080" depends_on: - nsy_lrs_1_db + healthcheck: + test: wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1 + interval: 5s + timeout: 5s + retries: 10 environment: LRSQL_API_KEY_DEFAULT: my_key LRSQL_API_SECRET_DEFAULT: my_secret @@ -62,6 +67,11 @@ services: - "8082:8080" depends_on: - nsy_lrs_2_db + healthcheck: + test: wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1 + interval: 5s + timeout: 5s + retries: 10 environment: LRSQL_API_KEY_DEFAULT: my_key LRSQL_API_SECRET_DEFAULT: my_secret @@ -95,6 +105,11 @@ services: - "8083:8080" depends_on: - nsy_lrs_3_db + healthcheck: + test: wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1 + interval: 5s + timeout: 5s + retries: 10 environment: LRSQL_API_KEY_DEFAULT: my_key LRSQL_API_SECRET_DEFAULT: my_secret @@ -130,6 +145,11 @@ services: - "8090:8080" depends_on: - txn_lrs_db + healthcheck: + test: wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1 + interval: 5s + timeout: 5s + retries: 10 environment: LRSQL_API_KEY_DEFAULT: my_key LRSQL_API_SECRET_DEFAULT: my_secret @@ -156,9 +176,12 @@ services: lrspipe1: image: yetanalytics/xapipe:latest depends_on: - - nsy_lrs_1 - - txn_lrs - - redis + nsy_lrs_1: + condition: service_healthy + txn_lrs: + condition: service_healthy + redis: + condition: service_started command: | -s redis --job-id nsy_lrs_1_sync @@ -175,9 +198,12 @@ services: lrspipe2: image: yetanalytics/xapipe:latest depends_on: - - nsy_lrs_2 - - txn_lrs - - redis + nsy_lrs_2: + condition: service_healthy + txn_lrs: + condition: service_healthy + redis: + condition: service_started command: | -s redis --job-id nsy_lrs_2_sync @@ -194,9 +220,12 @@ services: lrspipe3: image: yetanalytics/xapipe:latest depends_on: - - nsy_lrs_3 - - txn_lrs - - redis + nsy_lrs_3: + condition: service_healthy + txn_lrs: + condition: service_healthy + redis: + condition: service_started command: | -s redis --job-id nsy_lrs_3_sync From 66133fb3d87019d1ff2d6a3119d024d321330abf Mon Sep 17 00:00:00 2001 From: Cliff Casey Date: Wed, 25 Sep 2024 10:17:39 -0400 Subject: [PATCH 8/8] removed compose version --- dev-resources/tla-demo/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev-resources/tla-demo/docker-compose.yml b/dev-resources/tla-demo/docker-compose.yml index b69f8a2ba..964fe6e72 100644 --- a/dev-resources/tla-demo/docker-compose.yml +++ b/dev-resources/tla-demo/docker-compose.yml @@ -1,7 +1,5 @@ # Example Architecture with Two Noisy LRS' and one Transactional LRS -version: "3.9" - volumes: nsy_lrs_1_db_data: nsy_lrs_2_db_data: