diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index ef118a08420..72fd43b2035 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -77,6 +77,7 @@ rxdart saibatizoku seckey slotno +sqlfluff stevenj subosito tacho diff --git a/.sqlfluff b/.sqlfluff new file mode 100644 index 00000000000..3a43ddca946 --- /dev/null +++ b/.sqlfluff @@ -0,0 +1,24 @@ +# cspell: words capitalisation + +[sqlfluff] +dialect = postgres +large_file_skip_char_limit = 0 +max_line_length = 120 + +[sqlfluff:indentation] +tab_space_size = 2 + +[sqlfluff:rules:layout.long_lines] +ignore_comment_lines = True +ignore_comment_clauses = True + +[sqlfluff:rules:capitalisation.keywords] +capitalisation_policy = upper +[sqlfluff:rules:capitalisation.identifiers] +extended_capitalisation_policy = lower +[sqlfluff:rules:capitalisation.functions] +extended_capitalisation_policy = upper +[sqlfluff:rules:capitalisation.literals] +extended_capitalisation_policy = upper +[sqlfluff:rules:capitalisation.types] +extended_capitalisation_policy = upper \ No newline at end of file diff --git a/Earthfile b/Earthfile index 17217947de1..2933a0cf248 100644 --- a/Earthfile +++ b/Earthfile @@ -31,3 +31,12 @@ repo-docs: COPY --dir *.md LICENSE-APACHE LICENSE-MIT . SAVE ARTIFACT /repo repo + +repo-config-2: + # Create artifacts of config file we need to refer to in builders. + FROM scratch + + WORKDIR /repo + COPY --dir .sqlfluff . + + SAVE ARTIFACT /repo repo \ No newline at end of file diff --git a/catalyst-gateway/event-db/.sqlfluffignore b/catalyst-gateway/event-db/.sqlfluffignore new file mode 100644 index 00000000000..e4f16ddc7cc --- /dev/null +++ b/catalyst-gateway/event-db/.sqlfluffignore @@ -0,0 +1,4 @@ +# Ignore generated historic data files + +fund_*.sql +setup-db.sql \ No newline at end of file diff --git a/catalyst-gateway/event-db/Earthfile b/catalyst-gateway/event-db/Earthfile index 2e5ee86fd06..f8835b42df5 100644 --- a/catalyst-gateway/event-db/Earthfile +++ b/catalyst-gateway/event-db/Earthfile @@ -6,87 +6,56 @@ VERSION 0.7 # cspell: words -# Event db builder target. Prepares all necessary artifacts. -# Arguments: -# * with_historic_data: add historic data from the `historic_data` dir (default `true`). -# * with_test_data: add test data from the `test_data` dir (default `true`). +# Internal: builder is our Event db builder target. Prepares all necessary artifacts. +# CI target : dependency builder: - FROM github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.0.7+postgres-base + DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:fix/postgres-builders+BUILDER \ + --sqlfluff_cfg=./../../+repo-config/repo/.sqlfluff - ARG with_historic_data=true - ARG with_test_data=true - - WORKDIR /build - - COPY refinery.toml . - COPY --dir migrations . - RUN mkdir data - - IF [ $with_historic_data = true ] - COPY --dir historic_data . - - RUN python3 historic_data/fund_2/mk_fund2_sql.py historic_data/fund_2/fund2_database_encrypted.sqlite3 >> historic_data/fund_2.sql - RUN python3 historic_data/fund_3/mk_fund3_sql.py historic_data/fund_3/fund3_database_encrypted.sqlite3 >> historic_data/fund_3.sql - RUN python3 historic_data/fund_4/mk_fund4_sql.py historic_data/fund_4/fund4_database_encrypted.sqlite3 >> historic_data/fund_4.sql - RUN python3 historic_data/fund_5/mk_fund5_sql.py historic_data/fund_5/fund5_database_encrypted.sqlite3 >> historic_data/fund_5.sql - RUN python3 historic_data/fund_6/mk_fund6_sql.py historic_data/fund_6/fund6_database_encrypted.sqlite3 >> historic_data/fund_6.sql - RUN python3 historic_data/fund_7/mk_fund7_sql.py historic_data/fund_7/fund7_database_encrypted.sqlite3 >> historic_data/fund_7.sql - RUN python3 historic_data/fund_8/mk_fund8_sql.py historic_data/fund_8/fund8_database_encrypted.sqlite3 >> historic_data/fund_8.sql - RUN python3 historic_data/fund_9/mk_fund9_sql.py historic_data/fund_9/fund9_database_encrypted.sqlite3 >> historic_data/fund_9.sql - RUN python3 historic_data/fund_9/mk_fund9_voteplan_sql.py historic_data/fund_9/fund9_database_encrypted.sqlite3 >> historic_data/fund_9.sql - - RUN mv ./historic_data/*.sql ./data - END - - IF [ $with_test_data = true ] - COPY --dir test_data . - - RUN mv ./test_data/*.sql ./data - END - - DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.0.7+BUILDER + COPY ./../../+repo-config-2/repo/.sqlfluff . + COPY setup-db.sql /sql/ + COPY --dir json_schemas . +# check if the sql files are properly formatted and pass lint quality checks. +# CI target : true check: FROM +builder - DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.0.7+CHECK + DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:fix/postgres-builders+CHECK + +# format all SQL files in the current project. Local developers tool. +# CI target : false format: LOCALLY - DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.0.7+FORMAT --src=$(echo ${PWD}) + DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:fix/postgres-builders+FORMAT --src=$(echo ${PWD}/../../) -# Build event db docker image. -# Arguments: -# * tag: docker image `tag`. -# * registry: docker image `registry`. -# * with_historic_data: add historic data from the `historic_data` dir (default `true`). -# * with_test_data: add test data from the `test_data` dir (default `true`). +# build - an event db docker image. +# CI target : true build: - ARG with_historic_data=true - ARG with_test_data=true - ARG tag="latest" - ARG registry - - FROM +builder --with_historic_data=$with_historic_data --with_test_data=$with_test_data - - DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.0.7+BUILD --tag=$tag --registry=$registry --image_name=event-db - -test: - FROM github.com/input-output-hk/catalyst-ci/earthly/postgresql:v2.0.7+postgres-base - - COPY github.com/input-output-hk/catalyst-ci/earthly/utils:v2.0.7+shell-assert/assert.sh . - - COPY ./docker-compose.yml . - WITH DOCKER \ - --compose docker-compose.yml \ - --load event-db:latest=(+build --with_historic_data=false) \ - --service event-db \ - --allow-privileged - RUN sleep 5;\ - res=$(psql postgresql://catalyst-event-dev:CHANGE_ME@0.0.0.0:5432/CatalystEventDev -c "SELECT COUNT(*) FROM event");\ + FROM +builder - source assert.sh;\ - expected=$(printf " count \n-------\n 5\n(1 row)");\ - assert_eq "$expected" "$res" - END + DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:fix/postgres-builders+BUILD --image_name=event-db + DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:fix/postgres-builders+DOCS --image_name=event-db + +# test the event db database schema +# CI target : true +#test: +# FROM github.com/input-output-hk/catalyst-ci/earthly/postgresql:fix/postgres-builders+postgres-base + +# COPY github.com/input-output-hk/catalyst-ci/earthly/utils:fix/postgres-builders+shell-assert/assert.sh . + +# COPY ./docker-compose.yml . +# WITH DOCKER \ +# --compose docker-compose.yml \ +# --load event-db:latest=(+build --with_historic_data=false) \ +# --service event-db \ +# --allow-privileged +# RUN sleep 65;\ +# res=$(psql postgresql://catalyst-event-dev:CHANGE_ME@0.0.0.0:5432/CatalystEventDev -c "SELECT COUNT(*) FROM event");\ + +# source assert.sh;\ +# expected=$(printf " count \n-------\n 5\n(1 row)");\ +# assert_eq "$expected" "$res" +# END diff --git a/catalyst-gateway/event-db/diagrams.json b/catalyst-gateway/event-db/diagrams.json new file mode 100644 index 00000000000..3101afbf157 --- /dev/null +++ b/catalyst-gateway/event-db/diagrams.json @@ -0,0 +1,16 @@ +{ + "name": "Catalyst Gateway - Event DB", + "all_schema": { + "comments": true, + "included_tables": [], + "excluded_tables": [ + "refinery_schema_history" + ], + "column_description_wrap": 50, + "table_description_wrap": 120 + }, + "full_schema": { + "title": "Full Schema", + "comments": false + } +} \ No newline at end of file diff --git a/catalyst-gateway/event-db/json_schemas/.cspell.json b/catalyst-gateway/event-db/json_schemas/.cspell.json new file mode 100644 index 00000000000..9696eeafb41 --- /dev/null +++ b/catalyst-gateway/event-db/json_schemas/.cspell.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json", + "version": "0.2", + "enabled": true, + "language": "en,en-US", + "overrides": [ + { + "filename": "**/*.json", + "allowCompoundWords": true, + "words": [ + "sshuser", + "cexplorer", + "Cardano", + "jormungandr", + "dbsync" + ] + } + ] +} \ No newline at end of file diff --git a/catalyst-gateway/event-db/json_schemas/config/dbsync.json b/catalyst-gateway/event-db/json_schemas/config/dbsync.json new file mode 100644 index 00000000000..ca56f716b7a --- /dev/null +++ b/catalyst-gateway/event-db/json_schemas/config/dbsync.json @@ -0,0 +1,98 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$comment": "Custom URI schema: catalyst_schema:////", + "$id": "catalyst_schema://d899cd44-3513-487b-ab46-fdca662a724d/config/dbsync", + "title": "DBSync Connection", + "description": "DBSync connection configuration", + "type": "object", + "properties": { + "common": { + "$comment": "This provides the base settings for all networks. Anything thats common to all networks should be defined here.", + "$ref": "#/definitions/dbsync_conn" + }, + "networks": { + "$comment": "Individual networks defined here. Only need to define settings which differ from the common settings. Must define at least 1 network.", + "type": "object", + "properties": { + "mainnet": { + "$ref": "#/definitions/dbsync_conn" + }, + "preprod": { + "$ref": "#/definitions/dbsync_conn" + }, + "preview": { + "$ref": "#/definitions/dbsync_conn" + }, + "local": { + "$ref": "#/definitions/dbsync_conn" + } + }, + "anyOf": [ + { + "required": [ + "mainnet" + ] + }, + { + "required": [ + "preprod" + ] + }, + { + "required": [ + "preview" + ] + }, + { + "required": [ + "local" + ] + } + ] + } + }, + "required": [ + "common", + "networks" + ], + "additionalProperties": false, + "definitions": { + "dbsync_conn": { + "$comment": "DBSync connection", + "type": "object", + "properties": { + "host": { + "type": "string", + "description": "The hostname or IP address of the PostgreSQL server.", + "example": "localhost", + "default": "localhost" + }, + "port": { + "type": "integer", + "description": "The port number of the PostgreSQL server. Note, the default will be the ssh localPort if the connection is tunneled over SSH.", + "example": 5432, + "default": 5432 + }, + "database": { + "type": "string", + "description": "The name of the PostgreSQL database to connect to.", + "example": "cexplorer", + "default": "cexplorer" + }, + "username": { + "type": "string", + "description": "The username for authentication.", + "example": "cexplorer", + "default": "cexplorer" + }, + "password": { + "$comment": "If the password is not defined, it will be read from an env var called either DBSYNC_PWD_ and if thats not defined DBSYNC_PWD.", + "type": "string", + "description": "The password for authentication.", + "example": "password" + } + }, + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/catalyst-gateway/event-db/json_schemas/config/registration.json b/catalyst-gateway/event-db/json_schemas/config/registration.json new file mode 100644 index 00000000000..18b8ddff7ed --- /dev/null +++ b/catalyst-gateway/event-db/json_schemas/config/registration.json @@ -0,0 +1,86 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$comment": "Custom URI schema: catalyst_schema:////", + "$id": "catalyst_schema://62d614c0-97a7-41ec-a976-91294b8f4384/config/loadtest", + "title": "Registration Configuration", + "description": "Registration configuration", + "type": "object", + "properties": { + "common": { + "$comment": "This provides the base settings for all networks. Anything thats common to all networks should be defined here.", + "$ref": "#/definitions/registration" + }, + "networks": { + "$comment": "Individual networks defined here. Only need to define settings which differ from the common settings. Must define at least 1 network.", + "type": "object", + "properties": { + "mainnet": { + "$ref": "#/definitions/registration" + }, + "preprod": { + "$ref": "#/definitions/registration" + }, + "preview": { + "$ref": "#/definitions/registration" + }, + "local": { + "$ref": "#/definitions/registration" + } + }, + "anyOf": [ + { + "required": [ + "mainnet" + ] + }, + { + "required": [ + "preprod" + ] + }, + { + "required": [ + "preview" + ] + }, + { + "required": [ + "local" + ] + } + ] + } + }, + "required": [ + "common", + "networks" + ], + "additionalProperties": false, + "definitions": { + "registration": { + "$comment": "Registration Configuration", + "type": "object", + "properties": { + "loadtest": { + "type": "boolean", + "description": "Do we create LoadTest registrations when we import registrations for this network?", + "example": true, + "default": false + }, + "rollback_window": { + "type": "integer", + "description": "How many hours do we check for potential rollbacks when importing registrations for the network? Default is 7 Days, or 1 full clear Epoch.", + "example": 48, + "default": 168 + }, + "metadata_retention": { + "type": "integer", + "description": "How many days do we keep metadata for registrations in the database? Default is 6 months. 0 = Forever.", + "example": 30, + "default": 180 + } + }, + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/catalyst-gateway/event-db/json_schemas/event/data/catalyst_v1.json b/catalyst-gateway/event-db/json_schemas/event/data/catalyst_v1.json new file mode 100644 index 00000000000..ca6edfa16d4 --- /dev/null +++ b/catalyst-gateway/event-db/json_schemas/event/data/catalyst_v1.json @@ -0,0 +1,165 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$comment": "Custom URI schema: catalyst_schema:////", + "$id": "catalyst_schema://9c5df318-fa9a-4310-80fa-490f46d1cc43/event_data/catalyst_v1", + "title": "Catalyst V1 Event", + "description": "Traditional Project Catalyst Event Configuration", + "type": "object", + "properties": { + "timeline": { + "$comment": "allows additionalProperties so the schema can be extended in a backwards compatible way in the future", + "type": "object", + "description": "The timeline of the various stages of the event.", + "additionalProperties": true, + "properties": { + "registration_deadline": { + "description": "The deadline for registrations.\nThe Time (UTC) Registrations are taken from the Cardano blockchain.\nRegistrations after this date are not valid for voting on the event.\nNULL = Not yet defined or Not Applicable.'When the Voting power snapshot is set for.", + "type": "string", + "format": "date-time" + }, + "snapshot_stable": { + "description": "The Time (UTC) Registrations taken from Cardano blockchain are considered stable.\nThis is not the Time of the Registration Snapshot.\nThis is the time after which the registration snapshot will be stable.\nNULL = Not yet defined or Not Applicable.", + "type": "string", + "format": "date-time" + }, + "insight_sharing_start": { + "description": "TODO.\nNULL = Not yet defined.", + "type": "string", + "format": "date-time" + }, + "proposal_submission_start": { + "description": "The Time (UTC) proposals can start to be submitted for the event.\nNULL = Not yet defined, or Not applicable.", + "type": "string", + "format": "date-time" + }, + "refine_proposals_start": { + "description": "TODO.\nNULL = Not yet defined.", + "type": "string", + "format": "date-time" + }, + "finalize_proposals_start": { + "description": "The Time (UTC) when all proposals must be finalized by.\nNULL = Not yet defined, or Not applicable.", + "type": "string", + "format": "date-time" + }, + "proposal_assessment_start": { + "description": "The Time (UTC) when PA Assessors can start assessing proposals.\nNULL = Not yet defined, or Not applicable.", + "type": "string", + "format": "date-time" + }, + "assessment_qa_start": { + "description": "The Time (UTC) when vPA Assessors can start assessing assessments.\nNULL = Not yet defined, or Not applicable.", + "type": "string", + "format": "date-time" + }, + "voting_start": { + "description": "The earliest time that registered wallets with sufficient voting power can place votes in the event.\nNULL = Not yet defined.\nTypically this is aligned with Backing Start of the event.", + "type": "string", + "format": "date-time" + }, + "voting_end": { + "description": "The latest time that registered wallets with sufficient voting power can place votes in the event.\nNULL = Not yet defined.\nTypically aligned with backing End of the Event.", + "type": "string", + "format": "date-time" + }, + "tallying_end": { + "description": "The latest time that tallying the event can complete by.\nNULL = Not yet defined.", + "type": "string", + "format": "date-time" + } + } + }, + "registration": { + "$comment": "Allows additionalProperties so the schema can be extended in a backwards compatible way in the future.", + "type": "object", + "description": "The options which control the individual selection power of each participant in the event.", + "additionalProperties": true, + "properties": { + "stake_participation_threshold": { + "description": "The Minimum number of Lovelace needed to be staked at the time of snapshot, to be eligible to participate.\nNULL = Not yet defined.", + "type": "number", + "minimum": 0 + }, + "max_stake_percent": { + "description": "The Maximum Percentage of all registered staked ADA which can be used to participate in the event.", + "type": "number", + "minimum": 0, + "maximum": 100, + "default": 100 + }, + "network": { + "description": "The Cardano network the event allows registrations from. Can be one or more networks.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "mainnet", + "preprod", + "testnet" + ] + }, + "default": [ + "mainnet" + ] + } + } + }, + "rewards": { + "$comment": "Allows additionalProperties so the schema can be extended in a backwards compatible way in the future.", + "type": "object", + "description": "The options which control the reward distribution of the event.", + "additionalProperties": true, + "properties": { + "review": { + "description": "The total reward pool (in lovelace) to pay for community reviewers for their valid reviews of the proposals assigned to this event.", + "type": "integer", + "format": "int64", + "minimum": 0 + } + } + }, + "tally": { + "$comment": "Allows additionalProperties so the schema can be extended in a backwards compatible way in the future.", + "type": "object", + "description": "The options which control the results tally of the event.", + "additionalProperties": true, + "properties": { + "committee_size": { + "description": "The Size of the Tally committee.\n0 = No Committee, and all votes are therefore public.", + "type": "integer", + "format": "", + "minimum": 0, + "default": 0 + }, + "committee_threshold": { + "description": "The Minimum Size of the Tally committee to perform the Tally.\nMust be <= `committee_size`.\n0 is equivalent to ALL Committee members are required.", + "type": "integer", + "minimum": 0, + "default": 0 + } + } + }, + "jormungandr": { + "$comment": "Allows additionalProperties so the schema can be extended in a backwards compatible way in the future.", + "type": "object", + "description": "The Events Jormungandr configuration.", + "additionalProperties": true, + "properties": { + "block0": { + "description": "The BASE64 encoded copy of Block 0 used to start the Blockchain.\nIf not defined then the Blockchain is not yet configured.", + "type": "string", + "contentEncoding": "base64", + "contentMediaType": "application/octet-stream" + }, + "block0_hash": { + "description": "The BASE16 encoded hash of Block 0 used to start the Blockchain.\nIf not defined then the Blockchain is not yet configured.", + "type": "string", + "contentEncoding": "base16", + "contentMediaType": "application/octet-stream" + } + } + } + } +} \ No newline at end of file diff --git a/catalyst-gateway/event-db/json_schemas/event/description/multiline_text.json b/catalyst-gateway/event-db/json_schemas/event/description/multiline_text.json new file mode 100644 index 00000000000..1bd65e43be1 --- /dev/null +++ b/catalyst-gateway/event-db/json_schemas/event/description/multiline_text.json @@ -0,0 +1,11 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$comment": "Custom URI schema: catalyst_schema:////", + "$id": "catalyst_schema://d8f79cbc-9777-4fee-bbbe-397ce412a75c/event_description/multiline_text", + "title": "Simple Catalyst Event Description", + "description": "A simple multi-line description used for a Catalyst type event.", + "type": "string", + "contentEncoding": "utf-8", + "contentMediaType": "text/plain", + "format": "multiline" +} \ No newline at end of file diff --git a/catalyst-gateway/event-db/json_schemas/registration/cip36_stats.json b/catalyst-gateway/event-db/json_schemas/registration/cip36_stats.json new file mode 100644 index 00000000000..ce839213143 --- /dev/null +++ b/catalyst-gateway/event-db/json_schemas/registration/cip36_stats.json @@ -0,0 +1,119 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$comment": "Custom URI schema: catalyst_schema:////", + "$id": "catalyst_schema://fd5a2f8f-afb4-4cf7-ae6b-b7a370c85c82/registration/cip36_stats", + "title": "Individual Registration Statistics", + "description": "Statistical data associated with an individual CIP15/36 registration.", + "type": "object", + "properties": { + "loadtest": { + "type": "string", + "contentEncoding": "base16", + "contentMediaType": "application/octet-stream", + "description": "If present, then this is a Loadtest Synthetic registration. The field itself is the Private key used in a Loadtest to vote with this registration." + }, + "type": { + "type": "string", + "description": "The type of registration.", + "enum": [ + "Unknown", + "CIP-15", + "CIP-36" + ] + }, + "rollback": { + "type": "object", + "description": "Data about the last rollback that affected this registration. If not defined, this registration has never suffered from a rollback. Invalid registrations could be invalid because they were rolledback, and not re-included in the blockchain.", + "properties": { + "slot": { + "type": "integer", + "format": "int64", + "description": "The slot number the transaction was in, that got rolledback." + }, + "tip": { + "type": "integer", + "format": "int64", + "description": "The slot number of tip at the time of the rollback." + } + }, + "required": [ + "slot", + "tip" + ], + "additionalProperties": false + }, + "warnings": { + "type": "object", + "description": "List of warnings associated with the registration. These do not make it invalid but are signs of potential problems.", + "properties": { + "nonce_exceeds_slot": { + "type": "boolean", + "description": "Nonce is larger than the slot the registration appears in. This could make it difficult or impossible to supersede as registration.", + "default": false + }, + "reward_address_present": { + "type": "boolean", + "description": "The Payment address is a reward address. Payments are impossible to rewards type addresses.", + "default": false + } + } + }, + "errors": { + "type": "object", + "description": "List of errors associated with the registration which make it invalid.", + "properties": { + "registration": { + "type": "object", + "description": "List of errors associated with the registration itself.", + "properties": { + "missing": { + "type": "boolean", + "description": "No 61284 registration metadata found for the registration." + }, + "format": { + "type": "boolean", + "description": "Registration metadata was not formatted correctly and could not be parsed." + }, + "invalid_voting_key": { + "type": "boolean", + "description": "Voting key was not formatted correctly." + }, + "invalid_payment_address": { + "type": "boolean", + "description": "Payment address was not valid." + }, + "invalid_purpose": { + "type": "boolean", + "description": "The registration purpose was something other than 0." + }, + "multiple_voting_keys": { + "type": "boolean", + "description": "CIP-36 style multiple voting keys are not supported." + } + } + }, + "signature": { + "type": "object", + "description": "List of errors associated with the signature.", + "properties": { + "missing": { + "type": "boolean", + "description": "No 61285 signature metadata found for the registration." + }, + "format": { + "type": "boolean", + "description": "Signature metadata was not formatted correctly and could not be parsed." + }, + "invalid": { + "type": "boolean", + "description": "Signature failed to validate?" + } + } + } + } + } + }, + "required": [ + "type" + ] +} \ No newline at end of file diff --git a/catalyst-gateway/event-db/json_schemas/registration/update_stats.json b/catalyst-gateway/event-db/json_schemas/registration/update_stats.json new file mode 100644 index 00000000000..bcf64298c99 --- /dev/null +++ b/catalyst-gateway/event-db/json_schemas/registration/update_stats.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$comment": "Custom URI schema: catalyst_schema:////", + "$id": "catalyst_schema://0f917b13-afac-40d2-8263-b17ca8219914/registration/update_stats", + "title": "Registration Update Statistics", + "description": "Statistical data associated with the registration update.", + "type": "object", + "properties": { + "registrations_lost": { + "type": "integer", + "format": "int64", + "description": "The number of registrations that were lost during the update due to a rollback." + }, + "cip15_added": { + "type": "integer", + "format": "int64", + "description": "The number of CIP-15 registrations that were added during the update." + }, + "cip36_added": { + "type": "integer", + "format": "int64", + "description": "The number of CIP-36 registrations that were added during the update." + }, + "total_valid_registrations": { + "type": "integer", + "format": "int64", + "description": "The total number of registrations in the system as at this update." + }, + "total_unregistered_stake_addresses": { + "type": "integer", + "format": "int64", + "description": "The total number of stake_addresses not currently registered during this update." + }, + "total_staked_lovelace": { + "type": "integer", + "format": "int64", + "description": "The total of all staked lovelace in the system as at this update. This includes the total number of stake_addresses not currently registered during this update." + }, + "total_registered_lovelace": { + "type": "integer", + "format": "int64", + "description": "The total of all registered staked lovelace in the system as at this update." + } + } +} \ No newline at end of file diff --git a/catalyst-gateway/event-db/migrations/.sqlfluff b/catalyst-gateway/event-db/migrations/.sqlfluff new file mode 100644 index 00000000000..4e8310b4e9d --- /dev/null +++ b/catalyst-gateway/event-db/migrations/.sqlfluff @@ -0,0 +1,4 @@ +[sqlfluff] +# Supported dialects +# Set the dialect for all SQL files in this directory +dialect = postgres diff --git a/catalyst-gateway/event-db/migrations/V1__config_tables.sql b/catalyst-gateway/event-db/migrations/V1__config_tables.sql index d9e4a014047..fc22306e2a6 100644 --- a/catalyst-gateway/event-db/migrations/V1__config_tables.sql +++ b/catalyst-gateway/event-db/migrations/V1__config_tables.sql @@ -1,9 +1,14 @@ --- Catalyst Event Database +-- Catalyst Voices Database - Configuration Data +-- sqlfluff:dialect:postgres + +-- Title : Configuration Data + +-- Configuration Tables --- Version of the schema. +-- ------------------------------------------------------------------------------------------------- -CREATE TABLE IF NOT EXISTS refinery_schema_history -( +-- Version of the schema (Used by Refinery to manage migrations.). +CREATE TABLE IF NOT EXISTS refinery_schema_history ( version INTEGER NOT NULL PRIMARY KEY, name VARCHAR(255), applied_on VARCHAR(255), @@ -12,19 +17,70 @@ CREATE TABLE IF NOT EXISTS refinery_schema_history COMMENT ON TABLE refinery_schema_history IS 'History of Schema Updates to the Database. -Managed by the `refinery` cli tool. -'; +Managed by the `refinery` cli tool.'; + + +-- ------------------------------------------------------------------------------------------------- + +-- Json Schema Library +-- Json Schemas used to validate the contents of JSONB fields in this database. +-- * `id,type,name` matches the $id URI in the schema itself. +-- * The URI format is customized and is of the form `catalyst_schema:////` +-- * Schemas will be added here automatically during migration of the database, or interactively +-- * during operation of the system. +-- * They should match the schema they hold, and on read they should be validated. +-- * The code should refuse to serve or use any schema that does not match. +-- * +-- * id - This is unique and can uniquely identify any schema. +-- * type - This allows us to find all schemas of a known type. +-- * name - This is the unique name of the schema. `id` always equals the same `type/name`. +-- * for convention `type` and `name` string should only used a-z,0-9 and underscore. +-- * Dashes, symbols or upper case should not be used. +-- Catalyst Event Database +CREATE TABLE json_schema_type ( + id UUID PRIMARY KEY, + type TEXT NOT NULL, + name TEXT NOT NULL, + schema JSONB NOT NULL +); + +CREATE INDEX json_schema_type_idx ON json_schema_type ("type"); +CREATE UNIQUE INDEX json_schema_type_name_idx ON json_schema_type ( + "type", "name" +); + +COMMENT ON TABLE json_schema_type IS +'Library of defined json schemas used to validate JSONB field contents.'; + +COMMENT ON COLUMN json_schema_type.id IS +'Synthetic Unique ID for each json_schema_type (UUIDv4). +Must match the `UUID` component of the $id URI inside the schema.'; +COMMENT ON COLUMN json_schema_type.type IS +'The type of the json schema type. +eg. "event" +Must match the `type` component of the $id URI inside the schema.'; +COMMENT ON COLUMN json_schema_type.name IS +'The name of the json schema type. +eg. "catalyst_v1" +Must match the `name` component of the $id URI inside the schema.'; + +-- Known Schema Types are inserted when the Table which uses that type is created. +-- Or can be added by migrations as the database evolves. +-- They could also be added outside of the schema setup by inserting directly into the database. + +-- ------------------------------------------------------------------------------------------------- -- Config Table -- This table is looked up with three keys, `id`, `id2` and `id3` - -CREATE TABLE config -( +CREATE TABLE config ( row_id SERIAL PRIMARY KEY, id VARCHAR NOT NULL, id2 VARCHAR NOT NULL, id3 VARCHAR NOT NULL, - value JSONB NULL + value JSONB NULL, + value_schema UUID, + + FOREIGN KEY (value_schema) REFERENCES json_schema_type (id) ON DELETE CASCADE ); -- id+id2+id3 must be unique, they are a combined key. @@ -33,49 +89,48 @@ CREATE UNIQUE INDEX config_idx ON config (id, id2, id3); COMMENT ON TABLE config IS 'General JSON Configuration and Data Values. Defined Data Formats: - API Tokens: - `id` = "api_token" - `id2` = ` - `id3` = "" (Unused), - `value`->"name" = "", - `value`->"created" = , - `value`->"expires" = , - `value`->"perms" = {Permissions assigned to this api key} - - Community reviewers: - `id` = `email` - `id2` = `encrypted_password` - `id3` = `salt` - `value`->"role" = `role` - `value`->"name" = `name` - `value`->"anonymous_id" = `` - `value`->"force_reset" = "" - `value`->"active" = "" - - IdeaScale parameters: - `id` = "ideascale" - `id2` = "params" - `id3` = - `value`->"campaign_group_id" = - `value`->"stage_ids" = - - Event IdeaScale parameters: - `id` = "event" - `id2` = "ideascale_params" - `id3` = - `value`->"params_id" = + Currently None '; COMMENT ON COLUMN config.row_id IS -'Synthetic unique key. -Always lookup using id.'; -COMMENT ON COLUMN config.id IS 'The name/id of the general config value/variable'; +'Synthetic unique key. +Always lookup using `id.id2.id3`'; +COMMENT ON COLUMN config.id IS +'The name/id of the general config value/variable'; COMMENT ON COLUMN config.id2 IS -'2nd ID of the general config value. +'2nd ID of the general config value. Must be defined, use "" if not required.'; + COMMENT ON COLUMN config.id3 IS '3rd ID of the general config value. Must be defined, use "" if not required.'; -COMMENT ON COLUMN config.value IS 'The JSON value of the system variable id.id2.id3'; +COMMENT ON COLUMN config.value IS +'The JSON value of the system variable `id.id2.id3`'; +COMMENT ON COLUMN config.value_schema IS +'The Schema the Config Value conforms to. +The `value` field must conform to this schema.'; + +COMMENT ON INDEX config_idx IS +'We use three keys combined uniquely rather than forcing string concatenation +at the app level to allow for querying groups of data.'; + + +-- ------------------------------------------------------------------------------------------------- + +-- * Temporary. +-- * Insert known json schema manually until automated json schema migration scripting is added. +-- * This will be removed in the future. -COMMENT ON INDEX config_idx IS 'We use three keys combined uniquely rather than forcing string concatenation at the app level to allow for querying groups of data.'; +-- Add the Initial Schemas for configuration. +--INSERT INTO json_schema_type (id, type, name, schema) +--VALUES +--( +-- 'd899cd44-3513-487b-ab46-fdca662a724d', -- Fix the Schema ID so that it is consistent. +-- 'config', +-- 'dbsync', +-- (SELECT PG_READ_FILE('../json_schemas/config/dbsync.json'))::JSONB), ( +-- '62d614c0-97a7-41ec-a976-91294b8f4384', -- Fix the Schema ID so that it is consistent. +-- 'config', +-- 'registration', +-- (SELECT PG_READ_FILE('../json_schemas/config/registration.json'))::JSONB +--); diff --git a/catalyst-gateway/event-db/migrations/V2__event_tables.sql b/catalyst-gateway/event-db/migrations/V2__event_tables.sql index 76f5ab17017..8c9b5dd8ac3 100644 --- a/catalyst-gateway/event-db/migrations/V2__event_tables.sql +++ b/catalyst-gateway/event-db/migrations/V2__event_tables.sql @@ -1,150 +1,154 @@ --- Catalyst Event Database +-- Catalyst Voices Database - Event Data +-- sqlfluff:dialect:postgres +-- Title : Event Data --- Event Table - Defines each voting or decision event +-- `id` mapping. +-- * `id` is a UUID, in the past it was an auto incrementing value. +-- * it is changed to a UUID so that the data can be generated independently and it is more friendly +-- * and simpler for a decentralized or distributed system to safely create UUID than incremental number. +-- * However we need compatibility with the rpe-existing incremental numbers. +-- * Accordingly we will abuse the UUID format. +-- * A V8 UUID will be defined where the low 32 bits are 0s. +-- * If the ID is an incremental ID it will be mapped to this special uuid, by or-ing the incremental ID +-- * with the mapping UUID, creating a compatible UUID. +-- * As ALL autogenerated UUID's will be type 4, there is no possibility of collision. +-- +-- The Mapping UUID is defined as: `164fba58-31ff-8084-96cb-eb9d00000000` + + +-- Event Tables -CREATE TABLE event -( - row_id SERIAL PRIMARY KEY, +-- ------------------------------------------------------------------------------------------------- +CREATE TABLE event_type ( + id UUID PRIMARY KEY DEFAULT GEN_RANDOM_UUID(), name TEXT NOT NULL, - description TEXT NOT NULL, + description_schema UUID NOT NULL, + data_schema UUID NOT NULL, - registration_snapshot_time TIMESTAMP, - snapshot_start TIMESTAMP, - voting_power_threshold BIGINT, - max_voting_power_pct NUMERIC(6, 3) CONSTRAINT percentage CHECK ( - max_voting_power_pct <= 100 AND max_voting_power_pct >= 0 - ), + FOREIGN KEY (description_schema) REFERENCES json_schema_type ( + id + ) ON DELETE CASCADE, + FOREIGN KEY (data_schema) REFERENCES json_schema_type (id) ON DELETE CASCADE +); - review_rewards BIGINT, +CREATE UNIQUE INDEX event_type_name_idx ON event_type (name); + +COMMENT ON TABLE event_type IS +'The types of event which have been defined.'; + +COMMENT ON COLUMN event_type.id IS +'Synthetic Unique ID for each event_type (UUIDv4).'; +COMMENT ON COLUMN event_type.name IS +'The name of the event type. +eg. "Catalyst V1"'; +COMMENT ON COLUMN event_type.description_schema IS +'The JSON Schema which defines the structure of the data in the +`description` field in the event record.'; +COMMENT ON COLUMN event_type.data_schema IS +'The JSON Schema which defines the structure of the data in the +`extra_data` field in the event record.'; + +-- TODO: Would be better to read the schemas, extract the ID, and add or update new schemas. +-- Run as required after migrations. + +-- Add Event Schemas to the known schema types. +-- INSERT INTO json_schema_type_names (id) +-- VALUES +-- ('event_description'), -- Event Description schemas +-- ('event_data'); -- Event Data Schemas + +-- Add the Initial Schemas for events. +-- INSERT INTO json_schema_type (id, type, name, schema) +-- VALUES +--( +-- 'd899cd44-3513-487b-ab46-fdca662a724d', -- From the schema file. +-- 'event_description', +-- 'multiline_text', +-- ( +-- SELECT jsonb +-- FROM PG_READ_FILE('../json_schemas/event/description/multiline_text.json') +-- ) +--), +--( +-- '9c5df318-fa9a-4310-80fa-490f46d1cc43', -- From the schema file. +-- 'event_data', +-- 'catalyst_v1', +-- ( +-- SELECT jsonb +-- FROM PG_READ_FILE('../json_schemas/event/description/catalyst_v1.json') +-- ) +--); + +-- Define a Catalyst V1 Event. + +--INSERT INTO event_type (name, description_schema, data_schema) +--VALUES +--( +-- 'Catalyst V1', +-- 'd899cd44-3513-487b-ab46-fdca662a724d', +-- '9c5df318-fa9a-4310-80fa-490f46d1cc43' +--); + +-- ------------------------------------------------------------------------------------------------- +-- Event Table - Defines each voting or decision event +CREATE TABLE event ( + id UUID PRIMARY KEY DEFAULT GEN_RANDOM_UUID(), + -- The Organizer/Administrator of this event. + -- Update once RBAC is defined, as Organizer is an RBAC Role. + organizer TEXT NOT NULL, + type UUID REFERENCES event_type (id), + name TEXT NOT NULL, + description JSONB NOT NULL, start_time TIMESTAMP, + backing_start TIMESTAMP, + backing_end TIMESTAMP, end_time TIMESTAMP, + data JSONB NOT NULL +); - insight_sharing_start TIMESTAMP, - proposal_submission_start TIMESTAMP, - refine_proposals_start TIMESTAMP, - finalize_proposals_start TIMESTAMP, - proposal_assessment_start TIMESTAMP, - assessment_qa_start TIMESTAMP, - voting_start TIMESTAMP, - voting_end TIMESTAMP, - tallying_end TIMESTAMP, +CREATE UNIQUE INDEX event_name_idx ON event (name); - block0 BYTEA NULL, - block0_hash TEXT NULL, +COMMENT ON TABLE event IS +'The basic parameters of a related set of funding campaigns.'; - committee_size INTEGER NOT NULL, - committee_threshold INTEGER NOT NULL, +COMMENT ON COLUMN event.id IS +'Synthetic Unique ID for each event (UUIDv4).'; - extra JSONB, - cast_to JSONB -); - -CREATE UNIQUE INDEX event_name_idx ON event (name); +COMMENT ON COLUMN event.organizer IS +'Name of the Event Organizer. +Placeholder, this will need to be replaced +with a reference to an authorized organization.'; -COMMENT ON TABLE event IS 'The basic parameters of each voting/decision event.'; -COMMENT ON COLUMN event.row_id IS 'Synthetic Unique ID for each event.'; COMMENT ON COLUMN event.name IS 'The name of the event. eg. "Fund9" or "SVE1"'; + +COMMENT ON COLUMN event.type IS +'The type of the event.'; + COMMENT ON COLUMN event.description IS 'A detailed description of the purpose of the event. -eg. the events "Goal".'; -COMMENT ON COLUMN event.registration_snapshot_time IS -'The Time (UTC) Registrations are taken from Cardano main net. -Registrations after this date are not valid for voting on the event. -NULL = Not yet defined or Not Applicable.'; -COMMENT ON COLUMN event.snapshot_start IS -'The Time (UTC) Registrations taken from Cardano main net are considered stable. -This is not the Time of the Registration Snapshot, -This is the time after which the registration snapshot will be stable. -NULL = Not yet defined or Not Applicable.'; -COMMENT ON COLUMN event.voting_power_threshold IS -'The Minimum number of Lovelace staked at the time of snapshot, to be eligible to vote. -NULL = Not yet defined.'; -COMMENT ON COLUMN event.review_rewards IS 'The total reward pool to pay for community reviewers for their valid reviews of the proposals assigned to this event.'; +Must conform to the JSON Schema defined by +`event_type.description_schema.`'; COMMENT ON COLUMN event.start_time IS 'The time (UTC) the event starts. NULL = Not yet defined.'; -COMMENT ON COLUMN event.end_time IS -'The time (UTC) the event ends. -NULL = Not yet defined.'; -COMMENT ON COLUMN event.insight_sharing_start IS -'TODO. +COMMENT ON COLUMN event.backing_start IS +'The time (UTC) when backers may start backing the events campaign/s. +This must >= event.start_time. NULL = Not yet defined.'; -COMMENT ON COLUMN event.proposal_submission_start IS -'The Time (UTC) proposals can start to be submitted for the event. -NULL = Not yet defined, or Not applicable.'; -COMMENT ON COLUMN event.refine_proposals_start IS -'TODO. +COMMENT ON COLUMN event.backing_end IS +'The time (UTC) when backers may no longer back the events campaign/s. +This must > event.backing_start and <= event.end_time. NULL = Not yet defined.'; -COMMENT ON COLUMN event.finalize_proposals_start IS -'The Time (UTC) when all proposals must be finalized by. -NULL = Not yet defined, or Not applicable.'; -COMMENT ON COLUMN event.proposal_assessment_start IS -'The Time (UTC) when PA Assessors can start assessing proposals. -NULL = Not yet defined, or Not applicable.'; -COMMENT ON COLUMN event.assessment_qa_start IS -'The Time (UTC) when vPA Assessors can start assessing assessments. -NULL = Not yet defined, or Not applicable.'; -COMMENT ON COLUMN event.voting_start IS -'The earliest time that registered wallets with sufficient voting power can place votes in the event. -NULL = Not yet defined.'; -COMMENT ON COLUMN event.voting_end IS -'The latest time that registered wallets with sufficient voting power can place votes in the event. -NULL = Not yet defined.'; -COMMENT ON COLUMN event.tallying_end IS -'The latest time that tallying the event can complete by. +COMMENT ON COLUMN event.end_time IS +'The time (UTC) the event ends. +Must be >= event.backing_end. NULL = Not yet defined.'; - -COMMENT ON COLUMN event.block0 IS -'The copy of Block 0 used to start the Blockchain. -NULL = Blockchain not started yet.'; - -COMMENT ON COLUMN event.block0_hash IS -'The hash of block 0. -NULL = Blockchain not started yet.'; - -COMMENT ON COLUMN event.committee_size IS -'The size of the tally committee. -0 = No Committee, and all votes are therefore public.'; - -COMMENT ON COLUMN event.committee_threshold IS -'The minimum size of the tally committee to perform the tally. -Must be <= `committee_size`'; - -COMMENT ON COLUMN event.extra IS -'Json Map defining event specific extra data. -NULL = Not yet defined. -"url"."results" = a results URL, -"url"."survey" = a survey URL, -others can be defined as required.'; - -COMMENT ON COLUMN event.cast_to IS -'Json Map defining parameters which control where the vote is to be cast. -Multiple destinations can be defined simultaneously. -In this case the vote gets cast to all defined destinations. -`NULL` = Default Jormungandr Blockchain. -```jsonc -"jorm" : { // Voting on Jormungandr Blockchain - chain_id: , // Jormungandr chain id. Defaults to 0. - // Other parameters TBD. -}, -"cardano" : { // Voting on Cardano Directly - chain_id: , // 0 = pre-prod, 1 = mainnet. - // Other parameters TBD. -}, -"postgres" : { // Store votes in Web 2 postgres backed DB only. - url: "" - // Other parameters TBD. - // Note: Votes that arrive in the Cat1 system are always stored in the DB. - // This Option only allows the vote storage DB to be tuned. -}, -"cat2" : { // Store votes to the Catalyst 2.0 P2P Network. - gateway: "= 0 ), - event_id INTEGER NOT NULL, + event_id UUID NOT NULL, - FOREIGN KEY (event_id) REFERENCES event (row_id) ON DELETE CASCADE + FOREIGN KEY (event_id) REFERENCES event (id) ON DELETE CASCADE ); COMMENT ON TABLE reviewer_level IS @@ -92,7 +88,8 @@ This table represents all different types of reviewer`s levels, which is taken i COMMENT ON COLUMN reviewer_level.row_id IS 'Synthetic Unique Key'; COMMENT ON COLUMN reviewer_level.name IS 'Name of the reviewer level'; COMMENT ON COLUMN reviewer_level.total_reward_pct IS -'Total reviewer`s reward assigned to the specific level, which is defined as a percentage from the total pot of Community Review rewards (See `event.review_rewards` column).'; +'Total reviewer`s reward assigned to the specific level, which is defined as a percentage from the +total pot of Community Review rewards (See `event.review_rewards` column).'; COMMENT ON COLUMN reviewer_level.event_id IS 'The specific Event ID this review level is part of.'; -- community advisor reviews @@ -107,15 +104,6 @@ CREATE TABLE proposal_review ( assessor_level INTEGER, reward_address TEXT, - -- These fields are deprecated and WILL BE removed in a future migration. - -- They MUST only be used for Vit-SS compatibility. - impact_alignment_rating_given INTEGER, - impact_alignment_note VARCHAR, - feasibility_rating_given INTEGER, - feasibility_note VARCHAR, - auditability_rating_given INTEGER, - auditability_note VARCHAR, - ranking INTEGER, flags JSONB NULL, FOREIGN KEY (proposal_id) REFERENCES proposal (row_id) ON DELETE CASCADE, @@ -131,32 +119,6 @@ COMMENT ON COLUMN proposal_review.assessor IS 'Assessors Anonymized ID'; COMMENT ON COLUMN proposal_review.assessor_level IS 'Assessors level ID'; COMMENT ON COLUMN proposal_review.reward_address IS 'Assessors reward address'; -COMMENT ON COLUMN proposal_review.impact_alignment_rating_given IS -'The numeric rating assigned to the proposal by the assessor. -DEPRECATED: Only used for Vit-SS compatibility.'; -COMMENT ON COLUMN proposal_review.impact_alignment_note IS -'A note about why the impact rating was given. -DEPRECATED: Only used for Vit-SS compatibility.'; - -COMMENT ON COLUMN proposal_review.feasibility_rating_given IS -'The numeric feasibility rating given. -DEPRECATED: Only used for Vit-SS compatibility.'; -COMMENT ON COLUMN proposal_review.feasibility_note IS -'A note about why the feasibility rating was given. -DEPRECATED: Only used for Vit-SS compatibility.'; - -COMMENT ON COLUMN proposal_review.auditability_rating_given IS -'The numeric auditability rating given. -DEPRECATED: Only used for Vit-SS compatibility.'; -COMMENT ON COLUMN proposal_review.auditability_note IS -'A note about the auditability rating given. -DEPRECATED: Only used for Vit-SS compatibility.'; - -COMMENT ON COLUMN proposal_review.ranking IS -'Numeric Measure of quality of this review according to veteran community advisors. -DEPRECATED: Only used for Vit-SS compatibility. -'; - COMMENT ON COLUMN proposal_review.flags IS 'OPTIONAL: JSON Array which defines the flags raised for this review. Flags can be raised for different reasons and have different metadata. @@ -200,10 +162,10 @@ Each entry = -- Define known review metrics INSERT INTO review_metric (name, description, min, max, map) VALUES -('impact', 'Impact Rating', 0, 5, NULL), -('feasibility', 'Feasibility Rating', 0, 5, NULL), -('auditability', 'Auditability Rating', 0, 5, NULL), -('value', 'Value Proposition Rating', 0, 5, NULL), +('impact', 'Impact Rating', 0, 5, null), +('feasibility', 'Feasibility Rating', 0, 5, null), +('auditability', 'Auditability Rating', 0, 5, null), +('value', 'Value Proposition Rating', 0, 5, null), ('vpa_ranking', 'VPA Ranking of the review', 0, 3, ARRAY[ '{"name":"Excellent","desc":"Excellent Review"}', '{"name":"Good","desc":"Could be improved."}', diff --git a/catalyst-gateway/event-db/migrations/V5__vote_plan.sql b/catalyst-gateway/event-db/migrations/V5__vote_plan.sql index 48178183ed6..20d1b6120ec 100644 --- a/catalyst-gateway/event-db/migrations/V5__vote_plan.sql +++ b/catalyst-gateway/event-db/migrations/V5__vote_plan.sql @@ -1,5 +1,7 @@ -- Catalyst Event Database +-- Title : Vote Plan + -- Vote Plan Categories CREATE TABLE voteplan_category @@ -88,4 +90,5 @@ COMMENT ON TABLE proposal_voteplan IS 'Table to link Proposals to Vote plans in COMMENT ON COLUMN proposal_voteplan.row_id IS 'Synthetic ID of this Voteplan/Proposal M-M relationship.'; COMMENT ON COLUMN proposal_voteplan.proposal_id IS 'The link to the Proposal primary key that links to this voteplan.'; COMMENT ON COLUMN proposal_voteplan.voteplan_id IS 'The link to the Voteplan primary key that links to this proposal.'; -COMMENT ON COLUMN proposal_voteplan.bb_proposal_index IS 'The Index with the voteplan used by the voting ledger/bulletin board that references this proposal.'; +COMMENT ON COLUMN proposal_voteplan.bb_proposal_index IS +'The Index with the voteplan used by the voting ledger/bulletin board that references this proposal.'; diff --git a/catalyst-gateway/event-db/migrations/V6__registration.sql b/catalyst-gateway/event-db/migrations/V6__registration.sql new file mode 100644 index 00000000000..34816c92a70 --- /dev/null +++ b/catalyst-gateway/event-db/migrations/V6__registration.sql @@ -0,0 +1,347 @@ +-- Catalyst Voices Database - Role Registration Data +-- sqlfluff:dialect:postgres + +-- Title : Role Registration Data + +-- cspell: words utxo stxo +-- Configuration Tables + +-- ------------------------------------------------------------------------------------------------- + +-- Slot Index Table +CREATE TABLE cardano_slot_index ( + slot_no BIGINT NOT NULL, + network TEXT NOT NULL, + epoch_no BIGINT NOT NULL, + block_time TIMESTAMP NOT NULL, + block_hash BYTEA NOT NULL CHECK (LENGTH(block_hash) = 32), + + PRIMARY KEY (slot_no, network) +); + +CREATE INDEX cardano_slot_index_time_idx ON cardano_slot_index (block_time, network); +COMMENT ON INDEX cardano_slot_index_time_idx IS +'Index to allow us to efficiently lookup a slot by time for a particular network.'; + +CREATE INDEX cardano_slot_index_epoch_idx ON cardano_slot_index (epoch_no, network); +COMMENT ON INDEX cardano_slot_index_epoch_idx IS +'Index to allow us to efficiently lookup a slot by epoch for a particular network.'; + +COMMENT ON TABLE cardano_slot_index IS +'This is an index of cardano blockchain slots. +It allows us to quickly find data about every block in the cardano network. +This data is created when each block is first seen.'; + +COMMENT ON COLUMN cardano_slot_index.slot_no IS +'The slot number of the block. +This is the first half of the Primary key.'; +COMMENT ON COLUMN cardano_slot_index.network IS +'The Cardano network for this slot. +This is the second half of the primary key, as each network could use th same slot numbers.'; +COMMENT ON COLUMN cardano_slot_index.epoch_no IS +'The epoch number the slot appeared in.'; +COMMENT ON COLUMN cardano_slot_index.block_time IS +'The time of the slot/block.'; +COMMENT ON COLUMN cardano_slot_index.block_hash IS +'The hash of the block.'; + +-- ------------------------------------------------------------------------------------------------- + +-- Transaction Index Table +CREATE TABLE cardano_txn_index ( + id BYTEA NOT NULL PRIMARY KEY CHECK (LENGTH(id) = 32), + + slot_no BIGINT NULL, + network TEXT NOT NULL, + + FOREIGN KEY (slot_no, network) REFERENCES cardano_slot_index (slot_no, network) +); + +CREATE INDEX cardano_txn_index_idx ON cardano_txn_index (id, network); + +COMMENT ON INDEX cardano_txn_index_idx IS +'Index to allow us to efficiently get the slot a particular transaction is in.'; + +COMMENT ON TABLE cardano_txn_index IS +'This is an index of all transactions in the cardano network. +It allows us to quickly find a transaction by its id, and its slot number.'; + +COMMENT ON COLUMN cardano_txn_index.id IS +'The ID of the transaction. +This is a 32 Byte Hash.'; +COMMENT ON COLUMN cardano_txn_index.network IS +'The Cardano network for this transaction. +This is the second half of the primary key, as each network could use the same transactions.'; +COMMENT ON COLUMN cardano_txn_index.slot_no IS +'The slot number the transaction appeared in. +If this is NULL, then the Transaction is no longer in a known slot, due to a rollback. +Such transactions should be considered invalid until the appear in a new slot. +We only need to index transactions we care about and not every single transaction in the cardano network.'; + + +-- ------------------------------------------------------------------------------------------------- + +-- cardano update state table. +-- Keeps a record of each update to the Role Registration state data. +-- Used internally to track the updates to the database. +CREATE TABLE cardano_update_state ( + + id BIGSERIAL PRIMARY KEY, + + started TIMESTAMP NOT NULL, + ended TIMESTAMP NOT NULL, + updater_id TEXT NOT NULL, + + slot_no BIGINT NOT NULL, + network TEXT NOT NULL, + + update BOOLEAN NOT NULL, + rollback BOOLEAN NOT NULL, + + stats JSONB NOT NULL, + + FOREIGN KEY (slot_no, network) REFERENCES cardano_slot_index (slot_no, network) +); + +CREATE INDEX cardano_update_state_idx ON cardano_update_state (id, network); + +COMMENT ON INDEX cardano_update_state_idx IS +'Index to allow us to efficiently get find an update state record by its id. +This index can be used to find the latest state record for a particular network.'; + +CREATE INDEX cardano_update_state_time_idx ON cardano_update_state ( + started, network +); + +COMMENT ON INDEX cardano_update_state_time_idx IS +'Index to allow us to efficiently get find an update state record by time for a particular network.'; + + +COMMENT ON TABLE cardano_update_state IS +'A record of the updates to the Cardano Registration data state. +Every time the state is updated, a new record is created. +On update, an updating node, must check if the slot it is updating already exists. +If it does, it checks if the indexed block is the same (same hash). +If it is, it sets `update` to false, and just saves its update state with no further action. +This allows us to run multiple followers and update the database simultaneously. + +IF the hash is different, then we need to handle that, logic not yet defined... + +This table also serves as a central lock source for updates to the registration state +which must be atomic. +Should be accessed with a pattern like: + +```sql + BEGIN; + LOCK TABLE cardano_update_state IN ACCESS EXCLUSIVE MODE; + -- Read state, update any other tables as needed + INSERT INTO cardano_update_state SET ...; -- Set latest state + COMMIT; +``` +'; + +COMMENT ON COLUMN cardano_update_state.id IS +'Sequential ID of successive updates to the registration state data.'; +COMMENT ON COLUMN cardano_update_state.started IS +'The time the update started for this network.'; +COMMENT ON COLUMN cardano_update_state.ended IS +'The time the update was complete for this network.'; +COMMENT ON COLUMN cardano_update_state.updater_id IS +'The ID of the node which performed the update. +This helps us track which instance of the backend did which updates.'; +COMMENT ON COLUMN cardano_update_state.slot_no IS +'The slot_no this update was run for.'; +COMMENT ON COLUMN cardano_update_state.network IS +'The Cardano network that was updated. +As networks are independent and updates are event driven, only one network +will be updated at a time.'; +COMMENT ON COLUMN cardano_update_state.update IS +'True when this update updated any other tables. +False when a duplicate update was detected.'; +COMMENT ON COLUMN cardano_update_state.rollback IS +'True when this update is as a result of a rollback on-chain. +False when its a normal consecutive update.'; +COMMENT ON COLUMN cardano_update_state.stats IS +'A JSON stats record containing extra data about this update. +Must conform to Schema: + `catalyst_schema://0f917b13-afac-40d2-8263-b17ca8219914/registration/update_stats`.'; + +-- ------------------------------------------------------------------------------------------------- + +-- UTXO Table -- Unspent + Staked TX Outputs +-- Populated from the transactions in each block +CREATE TABLE cardano_utxo ( + tx_id BYTEA NOT NULL REFERENCES cardano_txn_index (id), + index INTEGER NOT NULL CHECK (index >= 0), + + value BIGINT NOT NULL, + asset JSONB NULL, + + stake_credential BYTEA NOT NULL, + + spent_tx_id BYTEA NULL REFERENCES cardano_txn_index (id), + + PRIMARY KEY (tx_id, index) +); + + +COMMENT ON TABLE cardano_utxo IS +'This table holds all UTXOs for any transaction which is tied to a stake address. +This data allows us to calculate staked ADA at any particular instant in time.'; + +COMMENT ON COLUMN cardano_utxo.tx_id IS +'The ID of the transaction containing the UTXO. +32 Byte Hash.'; +COMMENT ON COLUMN cardano_utxo.index IS +'The index of the UTXO within the transaction.'; + +COMMENT ON COLUMN cardano_utxo.value IS +'The value of the UTXO, in Lovelace if the asset is not defined.'; +COMMENT ON COLUMN cardano_utxo.asset IS +'The asset of the UTXO, if any. +NULL = Ada/Lovelace.'; + +COMMENT ON COLUMN cardano_utxo.stake_credential IS +'The stake credential of the address which owns the UTXO.'; + +COMMENT ON COLUMN cardano_utxo.spent_tx_id IS +'The ID of the transaction which Spent the TX Output. +If we consider this UTXO Spent will depend on when it was spent.'; + + +-- ------------------------------------------------------------------------------------------------- + +-- Rewards Table -- Earned Rewards +CREATE TABLE cardano_reward ( + slot_no BIGINT NOT NULL, -- First slot of the epoch following the epoch the rewards were earned for. + network TEXT NOT NULL, + stake_credential BYTEA NOT NULL, + + earned_epoch_no BIGINT NOT NULL, + + value BIGINT NOT NULL, + + PRIMARY KEY (slot_no, network, stake_credential), + FOREIGN KEY (slot_no, network) REFERENCES cardano_slot_index (slot_no, network) +); + +CREATE INDEX cardano_rewards_stake_credential_idx ON cardano_reward ( + stake_credential, slot_no +); + +COMMENT ON INDEX cardano_rewards_stake_credential_idx IS +'Index to allow us to efficiently lookup a set of Rewards by stake credential relative to a slot_no.'; + +COMMENT ON TABLE cardano_reward IS +'This table holds all earned rewards per stake address. +It is possible for a Stake Address to earn multiple rewards in the same epoch. +This record contains the Total of all rewards earned in the relevant epoch. +This data structure is preliminary pending the exact method of determining +the rewards earned by any particular stake address.'; + +COMMENT ON COLUMN cardano_reward.slot_no IS +'The slot number the rewards were earned for. +This is the first slot of the epoch following the epoch the rewards were earned for.'; +COMMENT ON COLUMN cardano_reward.network IS +'The Cardano network for this rewards.'; +COMMENT ON COLUMN cardano_reward.stake_credential IS +'The stake credential of the address who earned the rewards.'; +COMMENT ON COLUMN cardano_reward.earned_epoch_no IS +'The epoch number the rewards were earned for.'; +COMMENT ON COLUMN cardano_reward.value IS +'The value of the reward earned, in Lovelace'; + +-- ------------------------------------------------------------------------------------------------- + +-- Withdrawn Rewards Table -- Withdrawn Rewards +CREATE TABLE cardano_withdrawn_reward ( + slot_no BIGINT NOT NULL, + network TEXT NOT NULL, + stake_credential BYTEA NOT NULL, + value BIGINT NOT NULL, + + PRIMARY KEY (slot_no, network), + FOREIGN KEY (slot_no, network) REFERENCES cardano_slot_index (slot_no, network) +); + +COMMENT ON TABLE cardano_withdrawn_reward IS +'This table holds all withdrawn rewards data. +This makes it possible to accurately calculate the rewards which are still available for a specific Stake Address .'; + +COMMENT ON COLUMN cardano_withdrawn_reward.slot_no IS +'The slot number the rewards were withdrawn for.'; +COMMENT ON COLUMN cardano_withdrawn_reward.network IS +'The Cardano network this withdrawal occurred on.'; +COMMENT ON COLUMN cardano_withdrawn_reward.stake_credential IS +'The stake credential of the address who earned the rewards.'; +COMMENT ON COLUMN cardano_withdrawn_reward.value IS +'The value of the reward withdrawn, in Lovelace'; + +-- ------------------------------------------------------------------------------------------------- + +-- Cardano Voter Registrations Table -- Voter Registrations +CREATE TABLE cardano_voter_registration ( + tx_id BYTEA PRIMARY KEY NOT NULL REFERENCES cardano_txn_index (id), + + stake_credential BYTEA NULL, + public_voting_key BYTEA NULL, + payment_address BYTEA NULL, + nonce BIGINT NULL, + + metadata_61284 BYTEA NULL, -- We can purge metadata for valid registrations that are old to save storage space. + metadata_61285 BYTEA NULL, -- We can purge metadata for valid registrations that are old to save storage space. + + valid BOOLEAN NOT NULL DEFAULT false, + stats JSONB NULL + -- record rolled back in stats if the registration was lost during a rollback, its also invalid at this point. + -- Other stats we can record are is it a CIP-36 or CIP-15 registration format. + -- does it have a valid reward address but not a payment address, so we can't pay to it. + -- other flags about why the registration was invalid. + -- other flags about statistical data (if any). +); + +CREATE INDEX cardano_voter_registration_stake_credential_idx ON cardano_voter_registration ( + stake_credential, nonce, valid +); +COMMENT ON INDEX cardano_voter_registration_stake_credential_idx IS +'Optimize lookups for "stake_credential" or "stake_credential"+"nonce" or "stake_credential"+"nonce"+"valid".'; + +CREATE INDEX cardano_voter_registration_voting_key_idx ON cardano_voter_registration ( + public_voting_key, nonce, valid +); +COMMENT ON INDEX cardano_voter_registration_voting_key_idx IS +'Optimize lookups for "public_voting_key" or "public_voting_key"+"nonce" or "public_voting_key"+"nonce"+"valid".'; + +COMMENT ON TABLE cardano_voter_registration IS +'All CIP15/36 Voter Registrations that are on-chain. +This tables stores all found registrations, even if they are invalid, or have been rolled back.'; + +COMMENT ON COLUMN cardano_voter_registration.tx_id IS +'The Transaction hash of the Transaction holding the registration metadata. +This is used as the Primary Key because it is immutable in the face of potential rollbacks.'; + +COMMENT ON COLUMN cardano_voter_registration.stake_credential IS +'The stake credential of the address who registered.'; +COMMENT ON COLUMN cardano_voter_registration.public_voting_key IS +'The public voting key of the address who registered.'; +COMMENT ON COLUMN cardano_voter_registration.payment_address IS +'The payment address where any voter rewards associated with this registration will be sent.'; +COMMENT ON COLUMN cardano_voter_registration.nonce IS +'The nonce of the registration. Registrations for the same stake address with higher nonces have priority.'; + +COMMENT ON COLUMN cardano_voter_registration.metadata_61284 IS +'The raw metadata for the CIP-15/36 registration. +This data is optional, a parameter in config specifies how long raw registration metadata should be kept. +Outside this time, the Registration record will be kept, but the raw metadata will be purged.'; +COMMENT ON COLUMN cardano_voter_registration.metadata_61285 IS +'The metadata for the CIP-15/36 registration signature. +This data is optional, a parameter in config specifies how long signature metadata should be kept. +Outside this time, the Registration record will be kept, but the signature metadata will be purged.'; + +COMMENT ON COLUMN cardano_voter_registration.valid IS +'True if the registration is valid, false if the registration is invalid. +`stats` can be checked to determine WHY the registration is considered invalid.'; +COMMENT ON COLUMN cardano_voter_registration.stats IS +'Statistical information about the registration. +Must conform to Schema: + `catalyst_schema://fd5a2f8f-afb4-4cf7-ae6b-b7a370c85c82/registration/cip36_stats`.'; diff --git a/catalyst-gateway/event-db/migrations/V6__snapshot_tables.sql b/catalyst-gateway/event-db/migrations/V6__snapshot_tables.sql deleted file mode 100644 index 7e8115b629c..00000000000 --- a/catalyst-gateway/event-db/migrations/V6__snapshot_tables.sql +++ /dev/null @@ -1,129 +0,0 @@ --- Catalyst Event Database - --- Voting Power Snapshot Table - -CREATE TABLE snapshot ( - row_id SERIAL PRIMARY KEY, - event INTEGER NOT NULL UNIQUE, - as_at TIMESTAMP NOT NULL, - as_at_slotno INTEGER NOT NULL, - last_updated TIMESTAMP NOT NULL, - last_updated_slotno INTEGER NOT NULL, - - final BOOLEAN NOT NULL, - - dbsync_snapshot_cmd TEXT NULL, - dbsync_snapshot_params JSONB NULL, - dbsync_snapshot_data BYTEA NULL, - dbsync_snapshot_error BYTEA NULL, - dbsync_snapshot_unregistered BYTEA NULL, - - drep_data BYTEA NULL, - - catalyst_snapshot_cmd TEXT NULL, - catalyst_snapshot_params JSONB NULL, - catalyst_snapshot_data BYTEA NULL, - - FOREIGN KEY (event) REFERENCES event (row_id) ON DELETE CASCADE -); - -COMMENT ON TABLE snapshot IS -'Raw snapshot data for an event. -Only the latests snapshot per event is stored.'; -COMMENT ON COLUMN snapshot.event IS 'The event id this snapshot was for.'; -COMMENT ON COLUMN snapshot.as_at IS -'The time the snapshot was collected from dbsync. -This is the snapshot *DEADLINE*, i.e the time when registrations are final. -(Should be the slot time the dbsync_snapshot_cmd was run against.)'; -COMMENT ON COLUMN snapshot.last_updated IS -'The last time the snapshot was run -(Should be the latest block time taken from dbsync just before the snapshot was run.)'; -COMMENT ON COLUMN snapshot.final IS -'Is the snapshot Final? -No more updates will occur to this record once set.'; - -COMMENT ON COLUMN snapshot.dbsync_snapshot_cmd IS 'The name of the command run to collect the snapshot from dbsync.'; -COMMENT ON COLUMN snapshot.dbsync_snapshot_params IS 'The parameters passed to the command, each parameter is a key and its value is the value of the parameter.'; -COMMENT ON COLUMN snapshot.dbsync_snapshot_data IS -'The BROTLI COMPRESSED raw json result stored as BINARY from the dbsync snapshot. -(This is JSON data but we store as raw text to prevent any processing of it, and BROTLI compress to save space).'; -COMMENT ON COLUMN snapshot.dbsync_snapshot_error IS -'The BROTLI COMPRESSED raw json errors stored as BINARY from the dbsync snapshot. -(This is JSON data but we store as raw text to prevent any processing of it, and BROTLI compress to save space).'; -COMMENT ON COLUMN snapshot.dbsync_snapshot_unregistered IS -'The BROTLI COMPRESSED unregistered voting power stored as BINARY from the dbsync snapshot. -(This is JSON data but we store as raw text to prevent any processing of it, and BROTLI compress to save space).'; - -COMMENT ON COLUMN snapshot.drep_data IS -'The latest drep data obtained from GVC, and used in this snapshot calculation. -Should be in a form directly usable by the `catalyst_snapshot_cmd` -However, in order to save space this data is stored as BROTLI COMPRESSED BINARY.'; - -COMMENT ON COLUMN snapshot.catalyst_snapshot_cmd IS 'The actual name of the command run to produce the catalyst voting power snapshot.'; -COMMENT ON COLUMN snapshot.dbsync_snapshot_params IS 'The parameters passed to the command, each parameter is a key and its value is the value of the parameter.'; -COMMENT ON COLUMN snapshot.catalyst_snapshot_data IS -'The BROTLI COMPRESSED raw yaml result stored as BINARY from the catalyst snapshot calculation. -(This is YAML data but we store as raw text to prevent any processing of it, and BROTLI compress to save space).'; - --- voters - -CREATE TABLE voter ( - row_id SERIAL8 PRIMARY KEY, - - voting_key TEXT NOT NULL, - snapshot_id INTEGER NOT NULL, - voting_group TEXT NOT NULL, - - voting_power BIGINT NOT NULL, - - FOREIGN KEY (snapshot_id) REFERENCES snapshot (row_id) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX unique_voter_id ON voter ( - voting_key, voting_group, snapshot_id -); - -COMMENT ON TABLE voter IS 'Voting Power for every voting key.'; -COMMENT ON COLUMN voter.voting_key IS 'Either the voting key.'; -COMMENT ON COLUMN voter.snapshot_id IS 'The ID of the snapshot this record belongs to.'; -COMMENT ON COLUMN voter.voting_group IS 'The voter group the voter belongs to.'; -COMMENT ON COLUMN voter.voting_power IS 'Calculated Voting Power associated with this key.'; - --- contributions - -CREATE TABLE contribution ( - row_id SERIAL8 PRIMARY KEY, - - stake_public_key TEXT NOT NULL, - snapshot_id INTEGER NOT NULL, - - voting_key TEXT NULL, - voting_weight INTEGER NULL, - voting_key_idx INTEGER NULL, - value BIGINT NOT NULL, - - voting_group TEXT NOT NULL, - - -- each unique stake_public_key should have the same reward_address - reward_address TEXT NULL, - - FOREIGN KEY (snapshot_id) REFERENCES snapshot (row_id) ON DELETE CASCADE -); - -CREATE UNIQUE INDEX unique_contribution_id ON contribution ( - stake_public_key, voting_key, voting_group, snapshot_id -); - -COMMENT ON TABLE contribution IS 'Individual Contributions from stake public keys to voting keys.'; -COMMENT ON COLUMN contribution.row_id IS 'Synthetic Unique Row Key'; -COMMENT ON COLUMN contribution.stake_public_key IS 'The voters Stake Public Key'; -COMMENT ON COLUMN contribution.snapshot_id IS 'The snapshot this contribution was recorded from.'; - -COMMENT ON COLUMN contribution.voting_key IS 'The voting key. If this is NULL it is the raw staked ADA.'; -COMMENT ON COLUMN contribution.voting_weight IS 'The weight this voting key gets of the total.'; -COMMENT ON COLUMN contribution.voting_key_idx IS 'The index from 0 of the keys in the delegation array.'; -COMMENT ON COLUMN contribution.value IS 'The amount of ADA contributed to this voting key from the stake address'; - -COMMENT ON COLUMN contribution.voting_group IS 'The group this contribution goes to.'; - -COMMENT ON COLUMN contribution.reward_address IS 'Currently Unused. Should be the Stake Rewards address of the voter (currently unknown.)'; diff --git a/catalyst-gateway/event-db/migrations/V7__vote_tables.sql b/catalyst-gateway/event-db/migrations/V7__vote_tables.sql index ce92bbe57ac..8ef0f96e97a 100644 --- a/catalyst-gateway/event-db/migrations/V7__vote_tables.sql +++ b/catalyst-gateway/event-db/migrations/V7__vote_tables.sql @@ -1,4 +1,6 @@ --- Catalyst Event Database - VIT-SS Compatibility +-- Catalyst Event Database - Vote Storage + +-- Title : Vote Storage -- vote storage (replicates on-chain data for easy querying) @@ -7,13 +9,12 @@ CREATE TABLE ballot ( objective INTEGER NOT NULL, proposal INTEGER NULL, - voter INTEGER NOT NULL, + voter BYTEA NOT NULL, fragment_id TEXT NOT NULL, cast_at TIMESTAMP NOT NULL, choice SMALLINT NULL, raw_fragment BYTEA NOT NULL, - FOREIGN KEY (voter) REFERENCES voter (row_id) ON DELETE CASCADE, FOREIGN KEY (objective) REFERENCES objective (row_id) ON DELETE CASCADE, FOREIGN KEY (proposal) REFERENCES proposal (row_id) ON DELETE CASCADE ); @@ -23,7 +24,7 @@ CREATE UNIQUE INDEX ballot_objective_idx ON ballot (objective, fragment_id); COMMENT ON TABLE ballot IS 'All Ballots cast on an event.'; COMMENT ON COLUMN ballot.fragment_id IS 'Unique ID of this Ballot'; -COMMENT ON COLUMN ballot.voter IS 'Reference to the Voter who cast the ballot'; +COMMENT ON COLUMN ballot.voter IS 'Voters Voting Key who cast the ballot'; COMMENT ON COLUMN ballot.objective IS 'Reference to the Objective the ballot was for.'; COMMENT ON COLUMN ballot.proposal IS 'Reference to the Proposal the ballot was for. diff --git a/catalyst-gateway/event-db/migrations/V8__catalyst_automation.sql b/catalyst-gateway/event-db/migrations/V8__catalyst_automation.sql index dada1e5f450..53c42e4fa82 100644 --- a/catalyst-gateway/event-db/migrations/V8__catalyst_automation.sql +++ b/catalyst-gateway/event-db/migrations/V8__catalyst_automation.sql @@ -1,17 +1,19 @@ -- Catalyst Event Database +-- Title : Catalyst Automation + -- Voting Nodes Table - Defines nodes in the network -- This table is looked up by hostname and event CREATE TABLE voting_node ( hostname TEXT NOT NULL, - event INTEGER NOT NULL, + event UUID NOT NULL, pubkey TEXT NOT NULL, seckey TEXT NOT NULL, netkey TEXT NOT NULL, PRIMARY KEY (hostname, event), - FOREIGN KEY (event) REFERENCES event (row_id) ON DELETE CASCADE + FOREIGN KEY (event) REFERENCES event (id) ON DELETE CASCADE ); COMMENT ON TABLE voting_node IS @@ -20,7 +22,8 @@ It is used by nodes to self-bootstrap the blockchain.'; COMMENT ON COLUMN voting_node.hostname IS 'Unique hostname for the voting node.'; COMMENT ON COLUMN voting_node.event IS 'Unique event this node was configured for.'; COMMENT ON COLUMN voting_node.seckey IS 'Encrypted secret key from Ed25519 pair for the node. Used as the node secret.'; -COMMENT ON COLUMN voting_node.pubkey IS 'Public key from Ed25519 pair for the node. Used as consensus_leader_id when the node is a leader.'; +COMMENT ON COLUMN voting_node.pubkey IS +'Public key from Ed25519 pair for the node. Used as consensus_leader_id when the node is a leader.'; COMMENT ON COLUMN voting_node.netkey IS 'Encrypted Ed25519 secret key for the node. Used as the node p2p topology key.'; @@ -29,23 +32,27 @@ COMMENT ON COLUMN voting_node.netkey IS 'Encrypted Ed25519 secret key for the no CREATE TABLE tally_committee ( row_id SERIAL PRIMARY KEY, - event INTEGER NOT NULL UNIQUE, + event UUID NOT NULL UNIQUE, committee_pk TEXT NOT NULL, committee_id TEXT NOT NULL, member_crs TEXT, election_key TEXT, - FOREIGN KEY (event) REFERENCES event (row_id) ON DELETE CASCADE + FOREIGN KEY (event) REFERENCES event (id) ON DELETE CASCADE ); COMMENT ON TABLE tally_committee IS 'Table for storing data about the tally committee per voting event.'; COMMENT ON COLUMN tally_committee.row_id IS 'Unique ID for this committee member for this event.'; COMMENT ON COLUMN tally_committee.event IS 'The event this committee member is for.'; -COMMENT ON COLUMN tally_committee.committee_pk IS 'Encrypted private key for the committee wallet. This key can be used to get the committee public address.'; +COMMENT ON COLUMN tally_committee.committee_pk IS +'Encrypted private key for the committee wallet. This key can be used to get the committee public address.'; COMMENT ON COLUMN tally_committee.committee_id IS 'The hex-encoded public key for the committee wallet.'; -COMMENT ON COLUMN tally_committee.member_crs IS 'Encrypted Common Reference String shared in the creation of every set of committee member keys.'; -COMMENT ON COLUMN tally_committee.election_key IS 'Public key generated with all committee member public keys, and is used to encrypt votes. NULL if the event.committee_size is 0.'; +COMMENT ON COLUMN tally_committee.member_crs IS +'Encrypted Common Reference String shared in the creation of every set of committee member keys.'; +COMMENT ON COLUMN tally_committee.election_key IS +'Public key generated with all committee member public keys, and is used to encrypt votes. +NULL if the event.committee_size is 0.'; -- Committee Member Table - Stores data about the tally committee members @@ -67,7 +74,8 @@ CREATE TABLE committee_member ( COMMENT ON TABLE committee_member IS 'Table for storing data about the tally committee members.'; COMMENT ON COLUMN committee_member.row_id IS 'Unique ID for this committee member for this event.'; -COMMENT ON COLUMN committee_member.member_index IS 'the zero-based index of the member, ranging from 0 <= index < committee_size.'; +COMMENT ON COLUMN committee_member.member_index IS +'the zero-based index of the member, ranging from 0 <= index < committee_size.'; COMMENT ON COLUMN committee_member.committee IS 'The committee this member belongs to.'; COMMENT ON COLUMN committee_member.comm_pk IS 'Committee member communication public key.'; COMMENT ON COLUMN committee_member.comm_sk IS 'Encrypted committee member communication secret key.'; diff --git a/catalyst-gateway/event-db/migrations/V9__moderation_stage.sql b/catalyst-gateway/event-db/migrations/V9__moderation_stage.sql index 1d6c923d382..954fd9be072 100644 --- a/catalyst-gateway/event-db/migrations/V9__moderation_stage.sql +++ b/catalyst-gateway/event-db/migrations/V9__moderation_stage.sql @@ -1,5 +1,7 @@ -- Catalyst Event Database +-- Title : Moderation Stage Data + -- ModerationAllocation - Defines the relationship between users and proposals_reviews -- to describe the allocation of moderations that needs to be done. @@ -38,5 +40,6 @@ COMMENT ON TABLE moderation IS 'An individual moderation for a proposal review.' COMMENT ON COLUMN moderation.row_id IS 'Synthetic ID of this moderation.'; COMMENT ON COLUMN moderation.review_id IS 'The review the moderation is related to.'; COMMENT ON COLUMN moderation.user_id IS 'The user the moderation is submitted from.'; -COMMENT ON COLUMN moderation.classification IS 'The value used to describe the moderation (e.g. 0: excluded, 1: included).'; +COMMENT ON COLUMN moderation.classification IS +'The value used to describe the moderation (e.g. 0: excluded, 1: included).'; COMMENT ON COLUMN moderation.rationale IS 'The rationale for the given classification.'; diff --git a/catalyst-gateway/event-db/historic_data/README.md b/catalyst-gateway/event-db/old_seed/historic_data/README.md similarity index 100% rename from catalyst-gateway/event-db/historic_data/README.md rename to catalyst-gateway/event-db/old_seed/historic_data/README.md diff --git a/catalyst-gateway/event-db/historic_data/fund_0.sql b/catalyst-gateway/event-db/old_seed/historic_data/fund_0.sql similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_0.sql rename to catalyst-gateway/event-db/old_seed/historic_data/fund_0.sql diff --git a/catalyst-gateway/event-db/historic_data/fund_1.sql b/catalyst-gateway/event-db/old_seed/historic_data/fund_1.sql similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_1.sql rename to catalyst-gateway/event-db/old_seed/historic_data/fund_1.sql diff --git a/catalyst-gateway/event-db/historic_data/fund_2/block0.bin b/catalyst-gateway/event-db/old_seed/historic_data/fund_2/block0.bin similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_2/block0.bin rename to catalyst-gateway/event-db/old_seed/historic_data/fund_2/block0.bin diff --git a/catalyst-gateway/event-db/historic_data/fund_2/encrypt_fund2_sensitive_data.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_2/encrypt_fund2_sensitive_data.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_2/encrypt_fund2_sensitive_data.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_2/encrypt_fund2_sensitive_data.py diff --git a/catalyst-gateway/event-db/historic_data/fund_2/fund2_database_encrypted.sqlite3 b/catalyst-gateway/event-db/old_seed/historic_data/fund_2/fund2_database_encrypted.sqlite3 similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_2/fund2_database_encrypted.sqlite3 rename to catalyst-gateway/event-db/old_seed/historic_data/fund_2/fund2_database_encrypted.sqlite3 diff --git a/catalyst-gateway/event-db/historic_data/fund_2/genesis.yaml b/catalyst-gateway/event-db/old_seed/historic_data/fund_2/genesis.yaml similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_2/genesis.yaml rename to catalyst-gateway/event-db/old_seed/historic_data/fund_2/genesis.yaml diff --git a/catalyst-gateway/event-db/historic_data/fund_2/mk_fund2_sql.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_2/mk_fund2_sql.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_2/mk_fund2_sql.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_2/mk_fund2_sql.py diff --git a/catalyst-gateway/event-db/historic_data/fund_3/block0.bin b/catalyst-gateway/event-db/old_seed/historic_data/fund_3/block0.bin similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_3/block0.bin rename to catalyst-gateway/event-db/old_seed/historic_data/fund_3/block0.bin diff --git a/catalyst-gateway/event-db/historic_data/fund_3/encrypt_fund3_sensitive_data.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_3/encrypt_fund3_sensitive_data.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_3/encrypt_fund3_sensitive_data.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_3/encrypt_fund3_sensitive_data.py diff --git a/catalyst-gateway/event-db/historic_data/fund_3/fund3_database_encrypted.sqlite3 b/catalyst-gateway/event-db/old_seed/historic_data/fund_3/fund3_database_encrypted.sqlite3 similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_3/fund3_database_encrypted.sqlite3 rename to catalyst-gateway/event-db/old_seed/historic_data/fund_3/fund3_database_encrypted.sqlite3 diff --git a/catalyst-gateway/event-db/historic_data/fund_3/mk_fund3_sql.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_3/mk_fund3_sql.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_3/mk_fund3_sql.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_3/mk_fund3_sql.py diff --git a/catalyst-gateway/event-db/historic_data/fund_4/block0.bin b/catalyst-gateway/event-db/old_seed/historic_data/fund_4/block0.bin similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_4/block0.bin rename to catalyst-gateway/event-db/old_seed/historic_data/fund_4/block0.bin diff --git a/catalyst-gateway/event-db/historic_data/fund_4/encrypt_fund4_sensitive_data.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_4/encrypt_fund4_sensitive_data.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_4/encrypt_fund4_sensitive_data.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_4/encrypt_fund4_sensitive_data.py diff --git a/catalyst-gateway/event-db/historic_data/fund_4/fund4_database_encrypted.sqlite3 b/catalyst-gateway/event-db/old_seed/historic_data/fund_4/fund4_database_encrypted.sqlite3 similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_4/fund4_database_encrypted.sqlite3 rename to catalyst-gateway/event-db/old_seed/historic_data/fund_4/fund4_database_encrypted.sqlite3 diff --git a/catalyst-gateway/event-db/historic_data/fund_4/mk_fund4_sql.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_4/mk_fund4_sql.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_4/mk_fund4_sql.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_4/mk_fund4_sql.py diff --git a/catalyst-gateway/event-db/historic_data/fund_5/block0.bin b/catalyst-gateway/event-db/old_seed/historic_data/fund_5/block0.bin similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_5/block0.bin rename to catalyst-gateway/event-db/old_seed/historic_data/fund_5/block0.bin diff --git a/catalyst-gateway/event-db/historic_data/fund_5/encrypt_fund5_sensitive_data.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_5/encrypt_fund5_sensitive_data.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_5/encrypt_fund5_sensitive_data.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_5/encrypt_fund5_sensitive_data.py diff --git a/catalyst-gateway/event-db/historic_data/fund_5/fund5_database_encrypted.sqlite3 b/catalyst-gateway/event-db/old_seed/historic_data/fund_5/fund5_database_encrypted.sqlite3 similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_5/fund5_database_encrypted.sqlite3 rename to catalyst-gateway/event-db/old_seed/historic_data/fund_5/fund5_database_encrypted.sqlite3 diff --git a/catalyst-gateway/event-db/historic_data/fund_5/mk_fund5_sql.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_5/mk_fund5_sql.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_5/mk_fund5_sql.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_5/mk_fund5_sql.py diff --git a/catalyst-gateway/event-db/historic_data/fund_5/snapshot.json b/catalyst-gateway/event-db/old_seed/historic_data/fund_5/snapshot.json similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_5/snapshot.json rename to catalyst-gateway/event-db/old_seed/historic_data/fund_5/snapshot.json diff --git a/catalyst-gateway/event-db/historic_data/fund_6/block0.bin b/catalyst-gateway/event-db/old_seed/historic_data/fund_6/block0.bin similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_6/block0.bin rename to catalyst-gateway/event-db/old_seed/historic_data/fund_6/block0.bin diff --git a/catalyst-gateway/event-db/historic_data/fund_6/encrypt_fund6_sensitive_data.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_6/encrypt_fund6_sensitive_data.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_6/encrypt_fund6_sensitive_data.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_6/encrypt_fund6_sensitive_data.py diff --git a/catalyst-gateway/event-db/historic_data/fund_6/fund6_database_encrypted.sqlite3 b/catalyst-gateway/event-db/old_seed/historic_data/fund_6/fund6_database_encrypted.sqlite3 similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_6/fund6_database_encrypted.sqlite3 rename to catalyst-gateway/event-db/old_seed/historic_data/fund_6/fund6_database_encrypted.sqlite3 diff --git a/catalyst-gateway/event-db/historic_data/fund_6/mk_fund6_sql.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_6/mk_fund6_sql.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_6/mk_fund6_sql.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_6/mk_fund6_sql.py diff --git a/catalyst-gateway/event-db/historic_data/fund_7/block0.bin b/catalyst-gateway/event-db/old_seed/historic_data/fund_7/block0.bin similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_7/block0.bin rename to catalyst-gateway/event-db/old_seed/historic_data/fund_7/block0.bin diff --git a/catalyst-gateway/event-db/historic_data/fund_7/encrypt_fund7_sensitive_data.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_7/encrypt_fund7_sensitive_data.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_7/encrypt_fund7_sensitive_data.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_7/encrypt_fund7_sensitive_data.py diff --git a/catalyst-gateway/event-db/historic_data/fund_7/fund7_database_encrypted.sqlite3 b/catalyst-gateway/event-db/old_seed/historic_data/fund_7/fund7_database_encrypted.sqlite3 similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_7/fund7_database_encrypted.sqlite3 rename to catalyst-gateway/event-db/old_seed/historic_data/fund_7/fund7_database_encrypted.sqlite3 diff --git a/catalyst-gateway/event-db/historic_data/fund_7/mk_fund7_sql.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_7/mk_fund7_sql.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_7/mk_fund7_sql.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_7/mk_fund7_sql.py diff --git a/catalyst-gateway/event-db/historic_data/fund_8/block0.bin b/catalyst-gateway/event-db/old_seed/historic_data/fund_8/block0.bin similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_8/block0.bin rename to catalyst-gateway/event-db/old_seed/historic_data/fund_8/block0.bin diff --git a/catalyst-gateway/event-db/historic_data/fund_8/encrypt_fund8_sensitive_data.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_8/encrypt_fund8_sensitive_data.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_8/encrypt_fund8_sensitive_data.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_8/encrypt_fund8_sensitive_data.py diff --git a/catalyst-gateway/event-db/historic_data/fund_8/fund8_database_encrypted.sqlite3 b/catalyst-gateway/event-db/old_seed/historic_data/fund_8/fund8_database_encrypted.sqlite3 similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_8/fund8_database_encrypted.sqlite3 rename to catalyst-gateway/event-db/old_seed/historic_data/fund_8/fund8_database_encrypted.sqlite3 diff --git a/catalyst-gateway/event-db/historic_data/fund_8/mk_fund8_sql.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_8/mk_fund8_sql.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_8/mk_fund8_sql.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_8/mk_fund8_sql.py diff --git a/catalyst-gateway/event-db/historic_data/fund_9/block0.bin b/catalyst-gateway/event-db/old_seed/historic_data/fund_9/block0.bin similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_9/block0.bin rename to catalyst-gateway/event-db/old_seed/historic_data/fund_9/block0.bin diff --git a/catalyst-gateway/event-db/historic_data/fund_9/encrypt_fund9_sensitive_data.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_9/encrypt_fund9_sensitive_data.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_9/encrypt_fund9_sensitive_data.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_9/encrypt_fund9_sensitive_data.py diff --git a/catalyst-gateway/event-db/historic_data/fund_9/fund9_database_encrypted.sqlite3 b/catalyst-gateway/event-db/old_seed/historic_data/fund_9/fund9_database_encrypted.sqlite3 similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_9/fund9_database_encrypted.sqlite3 rename to catalyst-gateway/event-db/old_seed/historic_data/fund_9/fund9_database_encrypted.sqlite3 diff --git a/catalyst-gateway/event-db/historic_data/fund_9/mk_fund9_sql.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_9/mk_fund9_sql.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_9/mk_fund9_sql.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_9/mk_fund9_sql.py diff --git a/catalyst-gateway/event-db/historic_data/fund_9/mk_fund9_voteplan_sql.py b/catalyst-gateway/event-db/old_seed/historic_data/fund_9/mk_fund9_voteplan_sql.py similarity index 100% rename from catalyst-gateway/event-db/historic_data/fund_9/mk_fund9_voteplan_sql.py rename to catalyst-gateway/event-db/old_seed/historic_data/fund_9/mk_fund9_voteplan_sql.py diff --git a/catalyst-gateway/event-db/historic_data/sensitive-data-pub.pem b/catalyst-gateway/event-db/old_seed/historic_data/sensitive-data-pub.pem similarity index 100% rename from catalyst-gateway/event-db/historic_data/sensitive-data-pub.pem rename to catalyst-gateway/event-db/old_seed/historic_data/sensitive-data-pub.pem diff --git a/catalyst-gateway/event-db/stage_data/Readme.md b/catalyst-gateway/event-db/old_seed/stage_data/Readme.md similarity index 100% rename from catalyst-gateway/event-db/stage_data/Readme.md rename to catalyst-gateway/event-db/old_seed/stage_data/Readme.md diff --git a/catalyst-gateway/event-db/stage_data/dev/.gitignore b/catalyst-gateway/event-db/old_seed/stage_data/dev/.gitignore similarity index 100% rename from catalyst-gateway/event-db/stage_data/dev/.gitignore rename to catalyst-gateway/event-db/old_seed/stage_data/dev/.gitignore diff --git a/catalyst-gateway/event-db/stage_data/dev/00001_testfund_event.sql b/catalyst-gateway/event-db/old_seed/stage_data/dev/00001_testfund_event.sql similarity index 50% rename from catalyst-gateway/event-db/stage_data/dev/00001_testfund_event.sql rename to catalyst-gateway/event-db/old_seed/stage_data/dev/00001_testfund_event.sql index 5dbd0f1f632..2aeaeeac9af 100644 --- a/catalyst-gateway/event-db/stage_data/dev/00001_testfund_event.sql +++ b/catalyst-gateway/event-db/old_seed/stage_data/dev/00001_testfund_event.sql @@ -33,7 +33,7 @@ INSERT INTO event ( '1970-01-01 00:00:00', -- Snapshot Start. 450000000, -- Voting Power Threshold 1, -- Max Voting Power PCT - NULL, -- Review Rewards + null, -- Review Rewards '1970-01-01 00:00:00', -- Start Time '1970-01-01 00:00:00', -- End Time '1970-01-01 00:00:00', -- Insight Sharing Start @@ -45,36 +45,10 @@ INSERT INTO event ( '1970-01-01 00:00:00', -- Voting Starts '1970-01-01 00:00:00', -- Voting Ends '1970-01-01 00:00:00', -- Tallying Ends - NULL, -- Block 0 Data - NULL, -- Block 0 Hash + null, -- Block 0 Data + null, -- Block 0 Hash 1, -- Committee Size 1, -- Committee Threshold - NULL, -- Extra - NULL -- Cast to -) -ON CONFLICT (row_id) DO UPDATE -SET - name = EXCLUDED.name, - description = EXCLUDED.description, - registration_snapshot_time = EXCLUDED.registration_snapshot_time, - snapshot_start = EXCLUDED.snapshot_start, - voting_power_threshold = EXCLUDED.voting_power_threshold, - max_voting_power_pct = EXCLUDED.max_voting_power_pct, - review_rewards = EXCLUDED.review_rewards, - start_time = EXCLUDED.start_time, - end_time = EXCLUDED.end_time, - insight_sharing_start = EXCLUDED.insight_sharing_start, - proposal_submission_start = EXCLUDED.proposal_submission_start, - refine_proposals_start = EXCLUDED.refine_proposals_start, - finalize_proposals_start = EXCLUDED.finalize_proposals_start, - proposal_assessment_start = EXCLUDED.proposal_assessment_start, - assessment_qa_start = EXCLUDED.assessment_qa_start, - voting_start = EXCLUDED.voting_start, - voting_end = EXCLUDED.voting_end, - tallying_end = EXCLUDED.tallying_end, - block0 = EXCLUDED.block0, - block0_hash = EXCLUDED.block0_hash, - committee_size = EXCLUDED.committee_size, - committee_threshold = EXCLUDED.committee_threshold, - extra = EXCLUDED.extra, - cast_to = EXCLUDED.cast_to; + null, -- Extra + null -- Cast to +); diff --git a/catalyst-gateway/event-db/stage_data/dev/00002_testfund_ideascale_params.sql b/catalyst-gateway/event-db/old_seed/stage_data/dev/00002_testfund_ideascale_params.sql similarity index 99% rename from catalyst-gateway/event-db/stage_data/dev/00002_testfund_ideascale_params.sql rename to catalyst-gateway/event-db/old_seed/stage_data/dev/00002_testfund_ideascale_params.sql index 622f91caca1..1709e03eaca 100644 --- a/catalyst-gateway/event-db/stage_data/dev/00002_testfund_ideascale_params.sql +++ b/catalyst-gateway/event-db/old_seed/stage_data/dev/00002_testfund_ideascale_params.sql @@ -57,4 +57,4 @@ INSERT INTO config (id, id2, id3, value) VALUES ( '0', '{"params_id": "TestFund"}' ) ON CONFLICT (id, id2, id3) DO UPDATE -SET value = EXCLUDED.value; +SET value = excluded.value; diff --git a/catalyst-gateway/event-db/stage_data/local/.gitignore b/catalyst-gateway/event-db/old_seed/stage_data/local/.gitignore similarity index 100% rename from catalyst-gateway/event-db/stage_data/local/.gitignore rename to catalyst-gateway/event-db/old_seed/stage_data/local/.gitignore diff --git a/catalyst-gateway/event-db/stage_data/preprod/.gitignore b/catalyst-gateway/event-db/old_seed/stage_data/preprod/.gitignore similarity index 100% rename from catalyst-gateway/event-db/stage_data/preprod/.gitignore rename to catalyst-gateway/event-db/old_seed/stage_data/preprod/.gitignore diff --git a/catalyst-gateway/event-db/stage_data/prod/.gitignore b/catalyst-gateway/event-db/old_seed/stage_data/prod/.gitignore similarity index 100% rename from catalyst-gateway/event-db/stage_data/prod/.gitignore rename to catalyst-gateway/event-db/old_seed/stage_data/prod/.gitignore diff --git a/catalyst-gateway/event-db/stage_data/testnet/.gitignore b/catalyst-gateway/event-db/old_seed/stage_data/testnet/.gitignore similarity index 100% rename from catalyst-gateway/event-db/stage_data/testnet/.gitignore rename to catalyst-gateway/event-db/old_seed/stage_data/testnet/.gitignore diff --git a/catalyst-gateway/event-db/stage_data/testnet/00001_fund10_event.sql b/catalyst-gateway/event-db/old_seed/stage_data/testnet/00001_fund10_event.sql similarity index 50% rename from catalyst-gateway/event-db/stage_data/testnet/00001_fund10_event.sql rename to catalyst-gateway/event-db/old_seed/stage_data/testnet/00001_fund10_event.sql index 27e074daae7..2d227caa82c 100644 --- a/catalyst-gateway/event-db/stage_data/testnet/00001_fund10_event.sql +++ b/catalyst-gateway/event-db/old_seed/stage_data/testnet/00001_fund10_event.sql @@ -33,7 +33,7 @@ INSERT INTO event ( '2023-08-23 22:00:00', -- Snapshot Start. 450000000, -- Voting Power Threshold 1, -- Max Voting Power PCT - NULL, -- Review Rewards + null, -- Review Rewards '2023-06-16 19:56:00', -- Start Time '2023-09-18 00:00:00', -- End Time '2023-06-22 00:00:00', -- Insight Sharing Start @@ -45,36 +45,10 @@ INSERT INTO event ( '2023-08-31 11:00:00', -- Voting Starts '2023-09-14 11:00:00', -- Voting Ends '2023-09-18 11:00:00', -- Tallying Ends - NULL, -- Block 0 Data - NULL, -- Block 0 Hash + null, -- Block 0 Data + null, -- Block 0 Hash 1, -- Committee Size 1, -- Committee Threshold - NULL, -- Extra - NULL -- Cast to -) -ON CONFLICT (row_id) DO UPDATE -SET - name = EXCLUDED.name, - description = EXCLUDED.description, - registration_snapshot_time = EXCLUDED.registration_snapshot_time, - snapshot_start = EXCLUDED.snapshot_start, - voting_power_threshold = EXCLUDED.voting_power_threshold, - max_voting_power_pct = EXCLUDED.max_voting_power_pct, - review_rewards = EXCLUDED.review_rewards, - start_time = EXCLUDED.start_time, - end_time = EXCLUDED.end_time, - insight_sharing_start = EXCLUDED.insight_sharing_start, - proposal_submission_start = EXCLUDED.proposal_submission_start, - refine_proposals_start = EXCLUDED.refine_proposals_start, - finalize_proposals_start = EXCLUDED.finalize_proposals_start, - proposal_assessment_start = EXCLUDED.proposal_assessment_start, - assessment_qa_start = EXCLUDED.assessment_qa_start, - voting_start = EXCLUDED.voting_start, - voting_end = EXCLUDED.voting_end, - tallying_end = EXCLUDED.tallying_end, - block0 = EXCLUDED.block0, - block0_hash = EXCLUDED.block0_hash, - committee_size = EXCLUDED.committee_size, - committee_threshold = EXCLUDED.committee_threshold, - extra = EXCLUDED.extra, - cast_to = EXCLUDED.cast_to; + null, -- Extra + null -- Cast to +); diff --git a/catalyst-gateway/event-db/stage_data/testnet/00002_fund10_ideascale_params.sql b/catalyst-gateway/event-db/old_seed/stage_data/testnet/00002_fund10_ideascale_params.sql similarity index 99% rename from catalyst-gateway/event-db/stage_data/testnet/00002_fund10_ideascale_params.sql rename to catalyst-gateway/event-db/old_seed/stage_data/testnet/00002_fund10_ideascale_params.sql index 108dea796e0..a55f25f0514 100644 --- a/catalyst-gateway/event-db/stage_data/testnet/00002_fund10_ideascale_params.sql +++ b/catalyst-gateway/event-db/old_seed/stage_data/testnet/00002_fund10_ideascale_params.sql @@ -57,4 +57,4 @@ INSERT INTO config (id, id2, id3, value) VALUES ( '10', '{"params_id": "F10"}' ) ON CONFLICT (id, id2, id3) DO UPDATE -SET value = EXCLUDED.value; +SET value = excluded.value; diff --git a/catalyst-gateway/event-db/test_data/01_event_table.sql b/catalyst-gateway/event-db/old_seed/test_data/01_event_table.sql similarity index 96% rename from catalyst-gateway/event-db/test_data/01_event_table.sql rename to catalyst-gateway/event-db/old_seed/test_data/01_event_table.sql index 37fa58c55fa..2d47011e51a 100644 --- a/catalyst-gateway/event-db/test_data/01_event_table.sql +++ b/catalyst-gateway/event-db/old_seed/test_data/01_event_table.sql @@ -100,9 +100,9 @@ VALUES ( 5, 'Test Fund 5', 'Test Fund 5 description', - NULL, NULL, NULL, NULL, - NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, + null, null, null, null, + null, null, + null, null, null, null, null, null, null, null, null, + null, null, 6, 6 ); diff --git a/catalyst-gateway/event-db/seed/Readme.md b/catalyst-gateway/event-db/seed/Readme.md new file mode 100644 index 00000000000..f8065d2b270 --- /dev/null +++ b/catalyst-gateway/event-db/seed/Readme.md @@ -0,0 +1,3 @@ +# Seed data + +Seed data goes here, which each seed data set located in its own sub-directory. diff --git a/catalyst-gateway/event-db/setup-db.sql b/catalyst-gateway/event-db/setup-db.sql new file mode 100644 index 00000000000..07c4beeb85b --- /dev/null +++ b/catalyst-gateway/event-db/setup-db.sql @@ -0,0 +1,40 @@ +-- Initialize the Project Catalyst Event Database. + +-- cspell: words psql noqa + +-- This script requires a number of variables to be set. +-- They will default if not set externally. +-- These variables can be set on the "psql" command line. +-- Passwords may optionally be set by ENV Vars. +-- This script requires "psql" is run inside a POSIX compliant shell. + +-- VARIABLES: + +-- DISPLAY ALL VARIABLES +\echo VARIABLES: +\echo -> dbName ................. = :dbName +\echo -> dbDescription .......... = :dbDescription +\echo -> dbUser ................. = :dbUser +\echo -> dbUserPw ............... = xxxx +\echo test test + +-- Cleanup if we already ran this before. +DROP DATABASE IF EXISTS :"dbName"; -- noqa: PRS +DROP USER IF EXISTS :"dbUser"; -- noqa: PRS + +-- Create the test user we will use with the local dev database. +CREATE USER :"dbUser" WITH PASSWORD :'dbUserPw'; -- noqa: PRS + +-- Privileges for this user/role. +ALTER DEFAULT PRIVILEGES GRANT EXECUTE ON FUNCTIONS TO public; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT EXECUTE ON FUNCTIONS TO :"dbUser"; -- noqa: PRS + +GRANT EXECUTE ON FUNCTION PG_READ_FILE(TEXT, BIGINT, BIGINT, BOOLEAN) TO :"dbUser"; -- noqa: PRS + +-- Create the database. +CREATE DATABASE :"dbName" WITH OWNER :"dbUser"; -- noqa: PRS + +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO :"dbUser"; -- noqa: PRS + +COMMENT ON DATABASE :"dbName" IS :'dbDescription'; -- noqa: PRS diff --git a/docs/Earthfile b/docs/Earthfile index ce302f5fef9..b89e79dd6b9 100644 --- a/docs/Earthfile +++ b/docs/Earthfile @@ -13,6 +13,10 @@ src: # copy Rust docs COPY ./../catalyst-gateway+build-hosted/doc /docs/src/api/catalyst-gateway/rust-docs + # Copy docs we build in the postgres example. + COPY --dir ../catalyst-gateway/event-db+build/docs src/architecture/08_concepts/event_db + + # Build the docs here. docs: FROM +src diff --git a/docs/src/architecture/.pages b/docs/src/architecture/.pages index 870e4627de2..7ceda85940e 100644 --- a/docs/src/architecture/.pages +++ b/docs/src/architecture/.pages @@ -7,7 +7,7 @@ nav: - 05_building_block_view.md - 06_runtime_view.md - 07_deployment_view.md - - 08_concepts.md + - 08_concepts - 09_architecture_decisions - 10_quality_requirements.md - 11_technical_risks.md diff --git a/docs/src/architecture/08_concepts.md b/docs/src/architecture/08_concepts.md deleted file mode 100644 index 5fff894c370..00000000000 --- a/docs/src/architecture/08_concepts.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -icon: material/thought-bubble ---- - -# Cross-cutting Concepts - - - -## *~Concept 1~* - -... *~explanation~* - -## *~Concept 2~* - -... *~explanation~* - -… - -## *~Concept n~* - -... *~explanation~* diff --git a/docs/src/architecture/08_concepts/index.md b/docs/src/architecture/08_concepts/index.md new file mode 100644 index 00000000000..3b7e81c70a3 --- /dev/null +++ b/docs/src/architecture/08_concepts/index.md @@ -0,0 +1,9 @@ +--- +icon: material/thought-bubble +--- + +# Cross-cutting Concepts + + + +Each Concept will be documented in its own set of markdown files/images under this folder.