Update snapshot storing, replace sled db with the already used diesel…

… sql database (#275)

* add new Snapshot table

* add new VotingRegistration table

* add queries

* add Delegator table

* add delegations queries

* update

* add snapshot table creating, modify get_tags and update_from_raw_snapshot functions

* update db schema and models

* update migrations

* update

* refactor

* fix all tests

* remove unused code

* optimize

* fix lints

* fix clippy

* update snapshot-lib

* update db tables

* fix fmt

* update snapshot-lib, increase timeout for import_big_snapshot test

* fix clippy

* fix comments

* update comments

vit-servicing-station-lib/Cargo.toml

@@ -34,7 +34,6 @@ jormungandr-lib = { git = "https://github.com/input-output-hk/jormungandr", bran
 eccoxide = { git = "https://github.com/eugene-babichenko/eccoxide.git", branch = "fast-u64-scalar-mul", features = ["fast-u64-scalar-mul"], optional = true }
 http-zipkin = "0.3.0"
 notify = "=5.0.0-pre.11"
-sled = "0.34.7"
 
 # This solves building on windows when sqlite3lib is not installed or missing in the `$PATH`
 # as it happens with the github actions pipeline associated to this project.

vit-servicing-station-lib/migrations/2020-05-22-112032_setup_db/down.sql

@@ -11,4 +11,6 @@ DROP VIEW IF EXISTS full_proposals_info;
 DROP TABLE IF EXISTS goals;
 DROP TABLE IF EXISTS groups;
 DROP TABLE IF EXISTS votes;
-
+DROP TABLE IF EXISTS snapshots;
+DROP TABLE IF EXISTS voters;
+DROP TABLE IF EXISTS contributors;

vit-servicing-station-lib/migrations/2020-05-22-112032_setup_db/up.sql

@@ -145,6 +145,31 @@ create table votes (
     PRIMARY KEY("fragment_id")
 );
 
+create table snapshots (
+    tag TEXT NOT NULL primary key,
+    last_updated BIGINT NOT NULL
+);
+
+create table voters (
+    voting_key TEXT NOT NULL,
+    voting_power BIGINT NOT NULL,
+    voting_group TEXT NOT NULL,
+    snapshot_tag TEXT NOT NULL,
+    PRIMARY KEY(voting_key, voting_group, snapshot_tag)
+    FOREIGN KEY(snapshot_tag) REFERENCES snapshots(tag) ON DELETE CASCADE
+);
+
+create table contributors (
+    stake_public_key TEXT NOT NULL,
+    reward_address TEXT NOT NULL,
+    value BIGINT NOT NULL,
+    voting_key TEXT NOT NULL,
+    voting_group TEXT NOT NULL,
+    snapshot_tag TEXT NOT NULL,
+    PRIMARY KEY(stake_public_key, voting_key, voting_group, snapshot_tag),
+    FOREIGN KEY(snapshot_tag) REFERENCES snapshots(tag) ON DELETE CASCADE
+);
+
 CREATE VIEW full_proposals_info
 AS
 SELECT

vit-servicing-station-lib/src/db/models/funds.rs

@@ -35,6 +35,7 @@ pub struct Fund {
     #[serde(deserialize_with = "crate::utils::serde::deserialize_unix_timestamp_from_rfc3339")]
     #[serde(serialize_with = "crate::utils::serde::serialize_unix_timestamp_as_rfc3339")]
     pub registration_snapshot_time: i64,
+    #[serde(alias = "nextRegistrationSnapshotTime")]
     #[serde(deserialize_with = "crate::utils::serde::deserialize_unix_timestamp_from_rfc3339")]
     #[serde(serialize_with = "crate::utils::serde::serialize_unix_timestamp_as_rfc3339")]
     pub next_registration_snapshot_time: i64,

vit-servicing-station-lib/src/db/models/mod.rs

@@ -5,6 +5,7 @@ pub mod funds;
 pub mod goals;
 pub mod groups;
 pub mod proposals;
+pub mod snapshot;
 pub mod vote;
 pub mod vote_options;
 pub mod voteplans;

vit-servicing-station-lib/src/db/models/snapshot.rs

@@ -0,0 +1,88 @@
+use crate::db::schema::{contributors, snapshots, voters};
+use diesel::{ExpressionMethods, Insertable, Queryable};
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Queryable)]
+#[serde(rename_all = "camelCase")]
+pub struct Snapshot {
+    /// Tag - a unique identifier of the current snapshot
+    pub tag: String,
+    /// Timestamp for the latest update of the current snapshot
+    #[serde(deserialize_with = "crate::utils::serde::deserialize_unix_timestamp_from_rfc3339")]
+    #[serde(serialize_with = "crate::utils::serde::serialize_unix_timestamp_as_rfc3339")]
+    pub last_updated: i64,
+}
+
+impl Insertable<snapshots::table> for Snapshot {
+    type Values = (
+        diesel::dsl::Eq<snapshots::tag, String>,
+        diesel::dsl::Eq<snapshots::last_updated, i64>,
+    );
+
+    fn values(self) -> Self::Values {
+        (
+            snapshots::tag.eq(self.tag),
+            snapshots::last_updated.eq(self.last_updated),
+        )
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Queryable)]
+#[serde(rename_all = "camelCase")]
+pub struct Voter {
+    pub voting_key: String,
+    pub voting_power: i64,
+    pub voting_group: String,
+    pub snapshot_tag: String,
+}
+
+impl Insertable<voters::table> for Voter {
+    type Values = (
+        diesel::dsl::Eq<voters::voting_key, String>,
+        diesel::dsl::Eq<voters::voting_power, i64>,
+        diesel::dsl::Eq<voters::voting_group, String>,
+        diesel::dsl::Eq<voters::snapshot_tag, String>,
+    );
+
+    fn values(self) -> Self::Values {
+        (
+            voters::voting_key.eq(self.voting_key),
+            voters::voting_power.eq(self.voting_power),
+            voters::voting_group.eq(self.voting_group),
+            voters::snapshot_tag.eq(self.snapshot_tag),
+        )
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Queryable)]
+#[serde(rename_all = "camelCase")]
+pub struct Contributor {
+    pub stake_public_key: String,
+    pub reward_address: String,
+    pub value: i64,
+    pub voting_key: String,
+    pub voting_group: String,
+    pub snapshot_tag: String,
+}
+
+impl Insertable<contributors::table> for Contributor {
+    type Values = (
+        diesel::dsl::Eq<contributors::stake_public_key, String>,
+        diesel::dsl::Eq<contributors::reward_address, String>,
+        diesel::dsl::Eq<contributors::value, i64>,
+        diesel::dsl::Eq<contributors::voting_key, String>,
+        diesel::dsl::Eq<contributors::voting_group, String>,
+        diesel::dsl::Eq<contributors::snapshot_tag, String>,
+    );
+
+    fn values(self) -> Self::Values {
+        (
+            contributors::stake_public_key.eq(self.stake_public_key),
+            contributors::reward_address.eq(self.reward_address),
+            contributors::value.eq(self.value),
+            contributors::voting_key.eq(self.voting_key),
+            contributors::voting_group.eq(self.voting_group),
+            contributors::snapshot_tag.eq(self.snapshot_tag),
+        )
+    }
+}

vit-servicing-station-lib/src/db/queries/challenges.rs

@@ -14,7 +14,7 @@ pub async fn query_all_challenges(pool: &DbConnectionPool) -> Result<Vec<Challen
     tokio::task::spawn_blocking(move || {
         diesel::QueryDsl::order_by(
             challenges_dsl::challenges,
-            challenges::dsl::internal_id.asc(),
+            challenges_dsl::internal_id.asc(),
         )
         .load::<Challenge>(&db_conn)
         .map_err(|_| HandleError::InternalError("Error retrieving challenges".to_string()))
@@ -47,7 +47,7 @@ pub async fn query_challenges_by_fund_id(
             challenges_dsl::challenges,
             challenges_dsl::fund_id.eq(fund_id),
         )
-        .order_by(challenges::dsl::internal_id.asc())
+        .order_by(challenges_dsl::internal_id.asc())
         .load::<Challenge>(&db_conn)
         .map_err(|_e| HandleError::NotFound("Error loading challenges for fund id".to_string()))
     })

vit-servicing-station-lib/src/db/queries/funds.rs

@@ -137,7 +137,7 @@ pub fn insert_fund(fund: Fund, db_conn: &DbConnection) -> QueryResult<Fund> {
     funds::table.order(fund_dsl::id.desc()).first(db_conn)
 }
 
-pub async fn put_fund(fund: Fund, pool: &DbConnectionPool) -> Result<(), HandleError> {
+pub fn put_fund(fund: Fund, pool: &DbConnectionPool) -> Result<(), HandleError> {
     let db_conn = pool.get().map_err(HandleError::DatabaseError)?;
     diesel::replace_into(funds::table)
         .values(fund.values())

vit-servicing-station-lib/src/db/queries/mod.rs

@@ -6,5 +6,6 @@ pub mod goals;
 pub mod groups;
 pub mod proposals;
 pub mod search;
+pub mod snapshot;
 pub mod voteplans;
 pub mod votes;