diff --git a/.gitignore b/.gitignore index 0d60ef54ca..e6921a29aa 100644 --- a/.gitignore +++ b/.gitignore @@ -117,4 +117,4 @@ lefthook.yml treefmt.toml # local earthly Environments -local/ +./local/ diff --git a/containers/event-db-migrations/Earthfile b/containers/event-db-migrations/Earthfile index 43d89a9447..afc8885937 100644 --- a/containers/event-db-migrations/Earthfile +++ b/containers/event-db-migrations/Earthfile @@ -45,6 +45,7 @@ docker: COPY --dir ../../src/event-db+build/test_data ./test_data END COPY ../../src/event-db+build/refinery.toml . + COPY ../../src/event-db+build/stage_data ./stage_data VOLUME /eventdb/tmp COPY ./entry.sh . diff --git a/containers/event-db-migrations/entry.sh b/containers/event-db-migrations/entry.sh index f9ad18dfdc..0de7709ad4 100644 --- a/containers/event-db-migrations/entry.sh +++ b/containers/event-db-migrations/entry.sh @@ -19,6 +19,7 @@ # DB_USER_PASSWORD - The password of the database user # DB_SKIP_HISTORICAL_DATA - If set, historical data will not be added to the database (optional) # DB_SKIP_TEST_DATA - If set, test data will not be added to the database (optional) +# DB_SKIP_STAGE_DATA - If set, stage specific data will not be added to the database (optional) # ADMIN_ROLE_PASSWORD - The password of the cat_admin role for graphql # ADMIN_USER_PASSWORD - The password of the admin user for graphql # ANON_ROLE_PASSWORD - The password of the cat_anon role for graphql @@ -31,6 +32,7 @@ # SKIP_GRAPHQL_INIT - If set, graphql will not be initialized (optional) # DEBUG - If set, the script will print debug information (optional) # DEBUG_SLEEP - If set, the script will sleep for the specified number of seconds (optional) +# STAGE - The stage being run. Currently only controls if stage specific data is applied to the DB (optional) # --------------------------------------------------------------- set +x set -o errexit @@ -145,6 +147,20 @@ if [[ -z "${DB_SKIP_HISTORICAL_DATA:-}" ]]; then done < <(find ./historic_data -name '*.sql' -print0 | sort -z) fi +# Add stage specific data to the DB when initialized. +if [[ -z "${DB_SKIP_STAGE_DATA:-}" ]]; then + if [[ -n "$STAGE" ]]; then + STAGE_DATA="./stage_data/$STAGE" + else + STAGE_DATA="./stage_data/local" + fi + + while IFS= read -r -d '' file; do + echo "Adding stage specific data from $file" + psql -f "$file" + done < <(find "$STAGE_DATA" -name '*.sql' -print0 | sort -z) +fi + # Add test data if [[ -z "${DB_SKIP_TEST_DATA:-}" ]]; then while IFS= read -r -d '' file; do diff --git a/src/event-db/Earthfile b/src/event-db/Earthfile index c91448a36b..40b54f043c 100644 --- a/src/event-db/Earthfile +++ b/src/event-db/Earthfile @@ -12,6 +12,7 @@ build: COPY --dir migrations . # Copy the migrations directory to the working directory COPY --dir historic_data . # Copy the historic_data directory to the working directory COPY --dir test_data . # Copy the test_data directory to the working directory + COPY --dir stage_data . # Copy Stage specific data into the working directory. RUN python3 historic_data/fund_2/mk_fund2_sql.py historic_data/fund_2/fund2_database_encrypted.sqlite3 >> historic_data/fund_2.sql RUN python3 historic_data/fund_3/mk_fund3_sql.py historic_data/fund_3/fund3_database_encrypted.sqlite3 >> historic_data/fund_3.sql @@ -28,9 +29,10 @@ build: SAVE ARTIFACT migrations # Save the migrations directory as an artifact SAVE ARTIFACT historic_data # Save the historic_data directory as an artifact SAVE ARTIFACT test_data # Save the test_data directory as an artifact + SAVE ARTIFACT stage_data # Save the stage_data directory as an artifact docker-compose: - FROM scratch + FROM scratch COPY docker-compose.yml . SAVE ARTIFACT docker-compose.yml diff --git a/src/event-db/historic_data/fund_10.sql b/src/event-db/historic_data/fund_10.sql deleted file mode 100644 index 71ae1be461..0000000000 --- a/src/event-db/historic_data/fund_10.sql +++ /dev/null @@ -1,51 +0,0 @@ --- Data from Catalyst Fund 10 - Public Pilot Run - --- Purge all Fund 1 data before re-inserting it. -DELETE FROM event WHERE row_id = 10; - --- Create the Event record for Fund 0 - -INSERT INTO event -(row_id, name, description, - start_time, - end_time, - registration_snapshot_time, - snapshot_start, - voting_power_threshold, - max_voting_power_pct, - insight_sharing_start, - proposal_submission_start, - refine_proposals_start, - finalize_proposals_start, - proposal_assessment_start, - assessment_qa_start, - voting_start, - voting_end, - tallying_end, - block0, - block0_hash, - committee_size, - committee_threshold) -VALUES - -(10, 'Catalyst Fund 10', 'Test Fund 10 data', - '2020-08-08 00:00:00', -- Start Time - Date accurate, time not known. - '2020-09-22 00:00:00', -- End Time - Date accurate, time not known. - '2020-09-14 12:00:05', -- Registration Snapshot Time - Slot 8518514 - '2020-09-15 00:00:00', -- Snapshot Start - Date/time not known. - 1, -- Voting Power Threshold - Unknown, assume 1 - 100, -- Max Voting Power PCT - No max% threshold used in this fund. - NULL, -- Insight Sharing Start - None - '2020-08-08 00:00:00', -- Proposal Submission Start - Date accurate, time not known. - NULL, -- Refine Proposals Start - Date accurate, time not known. - '2020-09-11 00:00:00', -- Finalize Proposals Start - Date accurate, time not known. - NULL, -- Proposal Assessment Start - None - NULL, -- Assessment QA Start - None - '2020-09-17 00:00:00', -- Voting Starts - Date Accurate, time not known. - '2020-09-21 00:00:00', -- Voting Ends - Date Accurate, time not known. - '2020-09-22 23:59:00', -- Tallying Ends - Date Accurate, time not known. - NULL, -- Block 0 Data - Not Known - NULL, -- Block 0 Hash - Not Known - 0, -- Committee Size - Not Known - 0 -- Committee Threshold - Not Known - ); diff --git a/src/event-db/migrations/V1__config_tables.sql b/src/event-db/migrations/V1__config_tables.sql index 5706d389fb..39d86ecec8 100644 --- a/src/event-db/migrations/V1__config_tables.sql +++ b/src/event-db/migrations/V1__config_tables.sql @@ -41,7 +41,7 @@ Defined Data Formats: `value`->"created" = , `value`->"expires" = , `value`->"perms" = {Permissions assigned to this api key} - + Community reviewers: `id` = `email` `id2` = `encrypted_password` @@ -51,7 +51,19 @@ Defined Data Formats: `value`->"anonymous_id" = `` `value`->"force_reset" = "" `value`->"active" = "" - ... + + IdeaScale parameters: + `id` = "ideascale" + `id2` = "params" + `id3` = + `value`->"campaign_group_id" = + `value`->"stage_ids" = + + Event IdeaScale parameters: + `id` = "event" + `id2` = "ideascale_params" + `id3` = + `value`->"params_id" = '; COMMENT ON COLUMN config.row_id IS diff --git a/src/event-db/stage_data/Readme.md b/src/event-db/stage_data/Readme.md new file mode 100644 index 0000000000..1ea1055789 --- /dev/null +++ b/src/event-db/stage_data/Readme.md @@ -0,0 +1,12 @@ +# Stage Specific Data + +Subdirectories in this directory, that are named the same as a deployment stage will have their `.sql` files applied to the database when it is configured. + +* `dev` - Development Environment specific data. +* `testnet` - Test-Net Environment specific data. +* `preprod` - Preprod Environment specific data. +* `prod` - Production Environment specific data. +* `local` - Local Testing Environment specific data. Does not get checked in to git, local only. + +Each directory can only contain `*.sql` files. +They will be applied in sorted order. diff --git a/src/event-db/stage_data/dev/.gitignore b/src/event-db/stage_data/dev/.gitignore new file mode 100644 index 0000000000..f6893d3d19 --- /dev/null +++ b/src/event-db/stage_data/dev/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything in this directory +* +# Except these files +!.gitignore +!*.sql \ No newline at end of file diff --git a/src/event-db/stage_data/dev/00001_fund10_event.sql b/src/event-db/stage_data/dev/00001_fund10_event.sql new file mode 100644 index 0000000000..ba0040932f --- /dev/null +++ b/src/event-db/stage_data/dev/00001_fund10_event.sql @@ -0,0 +1,54 @@ +-- F10 +INSERT INTO event ( + row_id, + name, + description, + registration_snapshot_time, + snapshot_start, + voting_power_threshold, + max_voting_power_pct, + review_rewards, + start_time, + end_time, + insight_sharing_start, + proposal_submission_start, + refine_proposals_start, + finalize_proposals_start, + proposal_assessment_start, + assessment_qa_start, + voting_start, + voting_end, + tallying_end, + block0, + block0_hash, + committee_size, + committee_threshold, + extra, + cast_to +) VALUES ( + 10, + 'Fund 10', + 'Catalyst Dev Environment - Fund 10', + '2023-08-18 21:00:00', -- Registration Snapshot Time + '2023-08-23 22:00:00', -- Snapshot Start. + 450000000, -- Voting Power Threshold + 1, -- Max Voting Power PCT + NULL, -- Review Rewards + '2023-06-16 19:56:00', -- Start Time + '2023-09-18 00:00:00', -- End Time + '2023-06-22 00:00:00', -- Insight Sharing Start + '2023-06-22 00:00:00', -- Proposal Submission Start + '2023-06-22 00:00:00', -- Refine Proposals Start + '2023-07-13 00:00:00', -- Finalize Proposals Start + '2023-07-20 00:00:00', -- Proposal Assessment Start + '2023-08-10 00:00:00', -- Assessment QA Start + '2023-08-31 11:00:00', -- Voting Starts + '2023-09-14 11:00:00', -- Voting Ends + '2023-09-18 11:00:00', -- Tallying Ends + NULL, -- Block 0 Data + NULL, -- Block 0 Hash + 1, -- Committee Size + 1, -- Committee Threshold + NULL, -- Extra + NULL -- Cast to +); \ No newline at end of file diff --git a/src/event-db/stage_data/dev/00002_fund10_ideascale_params.sql b/src/event-db/stage_data/dev/00002_fund10_ideascale_params.sql new file mode 100644 index 0000000000..fcce721bec --- /dev/null +++ b/src/event-db/stage_data/dev/00002_fund10_ideascale_params.sql @@ -0,0 +1,18 @@ +-- Define F10 IdeaScale parameters. +INSERT INTO config (id, id2, id3, value) VALUES ( + 'ideascale', + 'params', + 'F10', + '{ + "campaign_group_id": 63, + "stage_ids": [4590, 4596, 4602, 4608, 4614, 4620, 4626, 4632, 4638, 4644, 4650, 4656, 4662] + }' +); + +-- Use F10 params for event with row_id = 10. +INSERT INTO config (id, id2, id3, value) VALUES ( + 'event', + 'ideascale_params', + '10', + '{"params_id": "F10"}' +); diff --git a/src/event-db/stage_data/local/.gitignore b/src/event-db/stage_data/local/.gitignore new file mode 100644 index 0000000000..86d0cb2726 --- /dev/null +++ b/src/event-db/stage_data/local/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore \ No newline at end of file diff --git a/src/event-db/stage_data/preprod/.gitignore b/src/event-db/stage_data/preprod/.gitignore new file mode 100644 index 0000000000..f6893d3d19 --- /dev/null +++ b/src/event-db/stage_data/preprod/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything in this directory +* +# Except these files +!.gitignore +!*.sql \ No newline at end of file diff --git a/src/event-db/stage_data/prod/.gitignore b/src/event-db/stage_data/prod/.gitignore new file mode 100644 index 0000000000..f6893d3d19 --- /dev/null +++ b/src/event-db/stage_data/prod/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything in this directory +* +# Except these files +!.gitignore +!*.sql \ No newline at end of file diff --git a/src/event-db/stage_data/testnet/.gitignore b/src/event-db/stage_data/testnet/.gitignore new file mode 100644 index 0000000000..f6893d3d19 --- /dev/null +++ b/src/event-db/stage_data/testnet/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything in this directory +* +# Except these files +!.gitignore +!*.sql \ No newline at end of file diff --git a/src/event-db/stage_data/testnet/00001_fund10_event.sql b/src/event-db/stage_data/testnet/00001_fund10_event.sql new file mode 100644 index 0000000000..8b6f0cb6c3 --- /dev/null +++ b/src/event-db/stage_data/testnet/00001_fund10_event.sql @@ -0,0 +1,54 @@ +-- F10 +INSERT INTO event ( + row_id, + name, + description, + registration_snapshot_time, + snapshot_start, + voting_power_threshold, + max_voting_power_pct, + review_rewards, + start_time, + end_time, + insight_sharing_start, + proposal_submission_start, + refine_proposals_start, + finalize_proposals_start, + proposal_assessment_start, + assessment_qa_start, + voting_start, + voting_end, + tallying_end, + block0, + block0_hash, + committee_size, + committee_threshold, + extra, + cast_to +) VALUES ( + 10, + 'Fund 10', + 'Catalyst Testnet - Fund 10', + '2023-08-18 21:00:00', -- Registration Snapshot Time + '2023-08-23 22:00:00', -- Snapshot Start. + 450000000, -- Voting Power Threshold + 1, -- Max Voting Power PCT + NULL, -- Review Rewards + '2023-06-16 19:56:00', -- Start Time + '2023-09-18 00:00:00', -- End Time + '2023-06-22 00:00:00', -- Insight Sharing Start + '2023-06-22 00:00:00', -- Proposal Submission Start + '2023-06-22 00:00:00', -- Refine Proposals Start + '2023-07-13 00:00:00', -- Finalize Proposals Start + '2023-07-20 00:00:00', -- Proposal Assessment Start + '2023-08-10 00:00:00', -- Assessment QA Start + '2023-08-31 11:00:00', -- Voting Starts + '2023-09-14 11:00:00', -- Voting Ends + '2023-09-18 11:00:00', -- Tallying Ends + NULL, -- Block 0 Data + NULL, -- Block 0 Hash + 1, -- Committee Size + 1, -- Committee Threshold + NULL, -- Extra + NULL -- Cast to +); \ No newline at end of file diff --git a/src/event-db/stage_data/testnet/00002_fund10_ideascale_params.sql b/src/event-db/stage_data/testnet/00002_fund10_ideascale_params.sql new file mode 100644 index 0000000000..fcce721bec --- /dev/null +++ b/src/event-db/stage_data/testnet/00002_fund10_ideascale_params.sql @@ -0,0 +1,18 @@ +-- Define F10 IdeaScale parameters. +INSERT INTO config (id, id2, id3, value) VALUES ( + 'ideascale', + 'params', + 'F10', + '{ + "campaign_group_id": 63, + "stage_ids": [4590, 4596, 4602, 4608, 4614, 4620, 4626, 4632, 4638, 4644, 4650, 4656, 4662] + }' +); + +-- Use F10 params for event with row_id = 10. +INSERT INTO config (id, id2, id3, value) VALUES ( + 'event', + 'ideascale_params', + '10', + '{"params_id": "F10"}' +);