feat: Stage specific data | NPG-7769

.gitignore

@@ -117,4 +117,4 @@ lefthook.yml
 treefmt.toml
 
 # local earthly Environments
-local/
+./local/

containers/event-db-migrations/Earthfile

@@ -45,6 +45,7 @@ docker:
         COPY --dir ../../src/event-db+build/test_data ./test_data
     END
     COPY ../../src/event-db+build/refinery.toml .
+    COPY ../../src/event-db+build/stage_data ./stage_data
 
     VOLUME /eventdb/tmp
     COPY ./entry.sh .

containers/event-db-migrations/entry.sh

@@ -19,6 +19,7 @@
 # DB_USER_PASSWORD - The password of the database user
 # DB_SKIP_HISTORICAL_DATA - If set, historical data will not be added to the database (optional)
 # DB_SKIP_TEST_DATA - If set, test data will not be added to the database (optional)
+# DB_SKIP_STAGE_DATA - If set, stage specific data will not be added to the database (optional)
 # ADMIN_ROLE_PASSWORD - The password of the cat_admin role for graphql
 # ADMIN_USER_PASSWORD - The password of the admin user for graphql
 # ANON_ROLE_PASSWORD - The password of the cat_anon role for graphql
@@ -31,6 +32,7 @@
 # SKIP_GRAPHQL_INIT - If set, graphql will not be initialized (optional)
 # DEBUG - If set, the script will print debug information (optional)
 # DEBUG_SLEEP - If set, the script will sleep for the specified number of seconds (optional)
+# STAGE - The stage being run.  Currently only controls if stage specific data is applied to the DB (optional)
 # ---------------------------------------------------------------
 set +x
 set -o errexit
@@ -145,6 +147,20 @@ if [[ -z "${DB_SKIP_HISTORICAL_DATA:-}" ]]; then
     done < <(find ./historic_data -name '*.sql' -print0 | sort -z)
 fi
 
+# Add stage specific data to the DB when initialized.
+if [[ -z "${DB_SKIP_STAGE_DATA:-}" ]]; then
+    if [[ -n "$STAGE" ]]; then
+        STAGE_DATA="./stage_data/$STAGE"
+    else
+        STAGE_DATA="./stage_data/local"
+    fi
+
+    while IFS= read -r -d '' file; do
+        echo "Adding stage specific data from $file"
+        psql -f "$file"
+    done < <(find "$STAGE_DATA" -name '*.sql' -print0 | sort -z)
+fi
+
 # Add test data
 if [[ -z "${DB_SKIP_TEST_DATA:-}" ]]; then
     while IFS= read -r -d '' file; do

src/event-db/Earthfile

@@ -12,6 +12,7 @@ build:
     COPY --dir migrations . # Copy the migrations directory to the working directory
     COPY --dir historic_data . # Copy the historic_data directory to the working directory
     COPY --dir test_data . # Copy the test_data directory to the working directory
+    COPY --dir stage_data . # Copy Stage specific data into the working directory.
 
     RUN python3 historic_data/fund_2/mk_fund2_sql.py historic_data/fund_2/fund2_database_encrypted.sqlite3 >> historic_data/fund_2.sql
     RUN python3 historic_data/fund_3/mk_fund3_sql.py historic_data/fund_3/fund3_database_encrypted.sqlite3 >> historic_data/fund_3.sql
@@ -28,9 +29,10 @@ build:
     SAVE ARTIFACT migrations # Save the migrations directory as an artifact
     SAVE ARTIFACT historic_data # Save the historic_data directory as an artifact
     SAVE ARTIFACT test_data # Save the test_data directory as an artifact
+    SAVE ARTIFACT stage_data # Save the stage_data directory as an artifact
 
 docker-compose:
-    FROM scratch 
+    FROM scratch
     COPY docker-compose.yml .
     SAVE ARTIFACT docker-compose.yml
 

src/event-db/migrations/V1__config_tables.sql

@@ -41,7 +41,7 @@ Defined  Data Formats:
     `value`->"created" = <Integer Unix Epoch when Token was created>,
     `value`->"expires" = <Integer Unix Epoch when Token will expire>,
     `value`->"perms" = {Permissions assigned to this api key}
-  
+
   Community reviewers:
     `id` = `email`
     `id2` = `encrypted_password`
@@ -51,7 +51,19 @@ Defined  Data Formats:
     `value`->"anonymous_id" = `<anonymous_id of the PA>`
     `value`->"force_reset" = "<bool used to force reset of password>"
     `value`->"active" = "<bool used to activate account>"
-    ...
+
+  IdeaScale parameters:
+    `id` = "ideascale"
+    `id2` = "params"
+    `id3` = <String identifying a fund, e.g. "F10">
+    `value`->"campaign_group_id" = <IdeaScale campaign group id>
+    `value`->"stage_ids" = <List of IdeaScale stage ids>
+
+  Event IdeaScale parameters:
+    `id` = "event"
+    `id2` = "ideascale_params"
+    `id3` = <Event row_id (as a string)>
+    `value`->"params_id" = <String identifier of the Ideascale parameters in the config table>
 ';
 
 COMMENT ON COLUMN config.row_id IS

src/event-db/stage_data/Readme.md

@@ -0,0 +1,12 @@
+# Stage Specific Data
+
+Subdirectories in this directory, that are named the same as a deployment stage will have their `.sql` files applied to the database when it is configured.
+
+* `dev` - Development Environment specific data.
+* `testnet` - Test-Net Environment specific data.
+* `preprod` - Preprod Environment specific data.
+* `prod` - Production Environment specific data.
+* `local` - Local Testing Environment specific data.  Does not get checked in to git, local only.
+
+Each directory can only contain `*.sql` files.
+They will be applied in sorted order.

src/event-db/stage_data/dev/.gitignore

@@ -0,0 +1,5 @@
+# Ignore everything in this directory
+*
+# Except these files
+!.gitignore
+!*.sql

src/event-db/stage_data/dev/00001_fund10_event.sql

@@ -0,0 +1,54 @@
+-- F10
+INSERT INTO event (
+    row_id,
+    name,
+    description,
+    registration_snapshot_time,
+    snapshot_start,
+    voting_power_threshold,
+    max_voting_power_pct,
+    review_rewards,
+    start_time,
+    end_time,
+    insight_sharing_start,
+    proposal_submission_start,
+    refine_proposals_start,
+    finalize_proposals_start,
+    proposal_assessment_start,
+    assessment_qa_start,
+    voting_start,
+    voting_end,
+    tallying_end,
+    block0,
+    block0_hash,
+    committee_size,
+    committee_threshold,
+    extra,
+    cast_to
+) VALUES (
+    10,
+    'Fund 10',
+    'Catalyst Dev Environment - Fund 10',
+    '2023-08-18 21:00:00',  -- Registration Snapshot Time
+    '2023-08-23 22:00:00',  -- Snapshot Start.
+    450000000,              -- Voting Power Threshold
+    1,                      -- Max Voting Power PCT
+    NULL,                   -- Review Rewards
+    '2023-06-16 19:56:00',  -- Start Time
+    '2023-09-18 00:00:00',  -- End Time
+    '2023-06-22 00:00:00',  -- Insight Sharing Start
+    '2023-06-22 00:00:00',  -- Proposal Submission Start
+    '2023-06-22 00:00:00',  -- Refine Proposals Start
+    '2023-07-13 00:00:00',  -- Finalize Proposals Start
+    '2023-07-20 00:00:00',  -- Proposal Assessment Start
+    '2023-08-10 00:00:00',  -- Assessment QA Start
+    '2023-08-31 11:00:00',  -- Voting Starts
+    '2023-09-14 11:00:00',  -- Voting Ends
+    '2023-09-18 11:00:00',  -- Tallying Ends
+    NULL,                   -- Block 0 Data
+    NULL,                   -- Block 0 Hash
+    1,                      -- Committee Size
+    1,                      -- Committee Threshold
+    NULL,                   -- Extra
+    NULL                    -- Cast to
+);

src/event-db/stage_data/dev/00002_fund10_ideascale_params.sql

@@ -0,0 +1,18 @@
+-- Define F10 IdeaScale parameters.
+INSERT INTO config (id, id2, id3, value) VALUES (
+    'ideascale',
+    'params',
+    'F10',
+    '{
+        "campaign_group_id": 63,
+        "stage_ids": [4590, 4596, 4602, 4608, 4614, 4620, 4626, 4632, 4638, 4644, 4650, 4656, 4662]
+     }'
+);
+
+-- Use F10 params for event with row_id = 10.
+INSERT INTO config (id, id2, id3, value) VALUES (
+    'event',
+    'ideascale_params',
+    '10',
+    '{"params_id": "F10"}'
+);

src/event-db/stage_data/local/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

src/event-db/stage_data/preprod/.gitignore

@@ -0,0 +1,5 @@
+# Ignore everything in this directory
+*
+# Except these files
+!.gitignore
+!*.sql

src/event-db/stage_data/prod/.gitignore

@@ -0,0 +1,5 @@
+# Ignore everything in this directory
+*
+# Except these files
+!.gitignore
+!*.sql

src/event-db/stage_data/testnet/.gitignore

@@ -0,0 +1,5 @@
+# Ignore everything in this directory
+*
+# Except these files
+!.gitignore
+!*.sql

src/event-db/stage_data/testnet/00001_fund10_event.sql

@@ -0,0 +1,54 @@
+-- F10
+INSERT INTO event (
+    row_id,
+    name,
+    description,
+    registration_snapshot_time,
+    snapshot_start,
+    voting_power_threshold,
+    max_voting_power_pct,
+    review_rewards,
+    start_time,
+    end_time,
+    insight_sharing_start,
+    proposal_submission_start,
+    refine_proposals_start,
+    finalize_proposals_start,
+    proposal_assessment_start,
+    assessment_qa_start,
+    voting_start,
+    voting_end,
+    tallying_end,
+    block0,
+    block0_hash,
+    committee_size,
+    committee_threshold,
+    extra,
+    cast_to
+) VALUES (
+    10,
+    'Fund 10',
+    'Catalyst Testnet - Fund 10',
+    '2023-08-18 21:00:00',  -- Registration Snapshot Time
+    '2023-08-23 22:00:00',  -- Snapshot Start.
+    450000000,              -- Voting Power Threshold
+    1,                      -- Max Voting Power PCT
+    NULL,                   -- Review Rewards
+    '2023-06-16 19:56:00',  -- Start Time
+    '2023-09-18 00:00:00',  -- End Time
+    '2023-06-22 00:00:00',  -- Insight Sharing Start
+    '2023-06-22 00:00:00',  -- Proposal Submission Start
+    '2023-06-22 00:00:00',  -- Refine Proposals Start
+    '2023-07-13 00:00:00',  -- Finalize Proposals Start
+    '2023-07-20 00:00:00',  -- Proposal Assessment Start
+    '2023-08-10 00:00:00',  -- Assessment QA Start
+    '2023-08-31 11:00:00',  -- Voting Starts
+    '2023-09-14 11:00:00',  -- Voting Ends
+    '2023-09-18 11:00:00',  -- Tallying Ends
+    NULL,                   -- Block 0 Data
+    NULL,                   -- Block 0 Hash
+    1,                      -- Committee Size
+    1,                      -- Committee Threshold
+    NULL,                   -- Extra
+    NULL                    -- Cast to
+);

src/event-db/stage_data/testnet/00002_fund10_ideascale_params.sql

@@ -0,0 +1,18 @@
+-- Define F10 IdeaScale parameters.
+INSERT INTO config (id, id2, id3, value) VALUES (
+    'ideascale',
+    'params',
+    'F10',
+    '{
+        "campaign_group_id": 63,
+        "stage_ids": [4590, 4596, 4602, 4608, 4614, 4620, 4626, 4632, 4638, 4644, 4650, 4656, 4662]
+     }'
+);
+
+-- Use F10 params for event with row_id = 10.
+INSERT INTO config (id, id2, id3, value) VALUES (
+    'event',
+    'ideascale_params',
+    '10',
+    '{"params_id": "F10"}'
+);