Implement --minimum-confidence parameter for vCA reward command

catalyst-toolbox/src/bin/cli/rewards/veterans.rs

@@ -62,6 +62,14 @@ pub struct VeteransRewards {
     /// if the first cutoff is selected then the first modifier is used.
     #[structopt(long, required = true)]
     reputation_agreement_rate_modifiers: Vec<Decimal>,
+
+    /// Value in range [0.5, 1]
+    /// The minimum confidence for a vCA ranking to be excluded from eligible rankings when in disagreement from simple majority.
+    /// Confidence is either `#FO / #Rankings` or `(#Excellent + #Good) / #Rankings` depending on the final ranking.
+    /// Simple majority is 50%. Qualified majority is 70%. Using 70% avoids punishing vCAs where this confidence is not clear.
+    /// 70% is because when #vCA == 3 confidence is only 66% and thus in this case, where there is just 1 vote in disagreement, all 3 vCAs get rewarded.
+    #[structopt(long)]
+    minimum_confidence: Decimal,
 }
 
 impl VeteransRewards {
@@ -77,6 +85,7 @@ impl VeteransRewards {
             rewards_agreement_rate_modifiers,
             reputation_agreement_rate_cutoffs,
             reputation_agreement_rate_modifiers,
+            minimum_confidence,
         } = self;
         let reviews: Vec<VeteranRankingRow> = csv::load_data_from_csv::<_, b','>(&from)?;
 
@@ -100,6 +109,10 @@ impl VeteransRewards {
             bail!("Expected rewards_agreement_rate_cutoffs to be descending");
         }
 
+        if minimum_confidence < Decimal::new(5, 1) || minimum_confidence > Decimal::ONE {
+            bail!("Expected minimum_confidence to range between .5 and 1");
+        }
+
         let results = veterans::calculate_veteran_advisors_incentives(
             &reviews,
             total_rewards,
@@ -113,6 +126,7 @@ impl VeteransRewards {
                 .into_iter()
                 .zip(reputation_agreement_rate_modifiers.into_iter())
                 .collect(),
+            minimum_confidence,
         );
 
         csv::dump_data_to_csv(rewards_to_csv_data(results).iter(), &to).unwrap();

catalyst-toolbox/src/rewards/veterans.rs

@@ -16,6 +16,15 @@ pub struct VeteranAdvisorIncentive {
     pub reputation: u64,
 }
 
+struct FinalRankingWithConfidence {
+    review_ranking: ReviewRanking,
+
+    /// This is to be used in conjunction with `ReviewRanking::is_positive()` to assess the
+    /// confidence of the boolean reply. It is either `#FO / #Rankings` or `(#Excellent + #Good) / #Rankings` depending on the final ranking.
+    /// For now we do not discriminate between Good and Excellent but this might change in the future.
+    confidence: Decimal,
+}
+
 pub type VcaRewards = HashMap<VeteranAdvisorId, VeteranAdvisorIncentive>;
 pub type EligibilityThresholds = std::ops::RangeInclusive<usize>;
 
@@ -25,18 +34,38 @@ pub type EligibilityThresholds = std::ops::RangeInclusive<usize>;
 // e.g. something like an expanded version of a AdvisorReviewRow
 // [proposal_id, advisor, ratings, ..(other fields from AdvisorReviewRow).., ranking (good/excellent/filtered out), vca]
 
-fn calc_final_ranking_per_review(rankings: &[impl Borrow<VeteranRankingRow>]) -> ReviewRanking {
+fn calc_final_ranking_with_confidence_per_review(
+    rankings: &[impl Borrow<VeteranRankingRow>],
+) -> FinalRankingWithConfidence {
     let rankings_majority = Decimal::from(rankings.len()) / Decimal::from(2);
     let ranks = rankings.iter().counts_by(|r| r.borrow().score());
 
-    match (ranks.get(&FilteredOut), ranks.get(&Excellent)) {
-        (Some(filtered_out), _) if Decimal::from(*filtered_out) >= rankings_majority => {
-            ReviewRanking::FilteredOut
+    match (
+        ranks.get(&Excellent),
+        ranks.get(&Good),
+        ranks.get(&FilteredOut),
+    ) {
+        (_, _, Some(filtered_out)) if Decimal::from(*filtered_out) >= rankings_majority => {
+            FinalRankingWithConfidence {
+                review_ranking: FilteredOut,
+                confidence: Decimal::from(*filtered_out) / Decimal::from(rankings.len()),
+            }
         }
-        (_, Some(excellent)) if Decimal::from(*excellent) > rankings_majority => {
-            ReviewRanking::Excellent
+        (Some(excellent), maybe_good, _) if Decimal::from(*excellent) > rankings_majority => {
+            FinalRankingWithConfidence {
+                review_ranking: Excellent,
+                confidence: (Decimal::from(maybe_good.copied().unwrap_or_default())
+                    + Decimal::from(*excellent))
+                    / Decimal::from(rankings.len()),
+            }
         }
-        _ => ReviewRanking::Good,
+        (maybe_excellent, Some(good), _) => FinalRankingWithConfidence {
+            review_ranking: Good,
+            confidence: (Decimal::from(maybe_excellent.copied().unwrap_or_default())
+                + Decimal::from(*good))
+                / Decimal::from(rankings.len()),
+        },
+        _ => unreachable!(),
     }
 }
 
@@ -84,12 +113,18 @@ pub fn calculate_veteran_advisors_incentives(
     reputation_thresholds: EligibilityThresholds,
     rewards_mod_args: Vec<(Decimal, Decimal)>,
     reputation_mod_args: Vec<(Decimal, Decimal)>,
+    minimum_confidence: Decimal,
 ) -> HashMap<VeteranAdvisorId, VeteranAdvisorIncentive> {
-    let final_rankings_per_review = veteran_rankings
+    let final_rankings_with_confidence_per_review = veteran_rankings
         .iter()
         .into_group_map_by(|ranking| ranking.review_id())
         .into_iter()
-        .map(|(review, rankings)| (review, calc_final_ranking_per_review(&rankings)))
+        .map(|(review, rankings)| {
+            (
+                review,
+                calc_final_ranking_with_confidence_per_review(&rankings),
+            )
+        })
         .collect::<BTreeMap<_, _>>();
 
     let rankings_per_vca = veteran_rankings
@@ -99,11 +134,13 @@ pub fn calculate_veteran_advisors_incentives(
     let eligible_rankings_per_vca = veteran_rankings
         .iter()
         .filter(|ranking| {
-            final_rankings_per_review
+            let final_ranking_with_confidence = final_rankings_with_confidence_per_review
                 .get(&ranking.review_id())
-                .unwrap()
-                .is_positive()
+                .unwrap();
+
+            final_ranking_with_confidence.review_ranking.is_positive()
                 == ranking.score().is_positive()
+                || final_ranking_with_confidence.confidence < minimum_confidence
         })
         .counts_by(|ranking| ranking.vca.clone());
 
@@ -156,6 +193,8 @@ mod tests {
     const VCA_1: &str = "vca1";
     const VCA_2: &str = "vca2";
     const VCA_3: &str = "vca3";
+    const SIMPLE_MAJORITY_CONFIDENCE: Decimal = dec!(.5);
+    const QUALIFIED_MAJORITY_CONFIDENCE: Decimal = dec!(.7);
 
     struct RandomIterator;
     impl Iterator for RandomIterator {
@@ -188,23 +227,36 @@ mod tests {
     #[test]
     fn final_ranking_is_correct() {
         assert!(matches!(
-            calc_final_ranking_per_review(&gen_dummy_rankings("".into(), 5, 5, 5, RandomIterator),),
-            ReviewRanking::Good
+            calc_final_ranking_with_confidence_per_review(&gen_dummy_rankings("".into(), 5, 5, 5, RandomIterator)),
+            FinalRankingWithConfidence {
+                review_ranking: Good,
+                confidence
+            } if confidence == (dec!(10) / dec!(15))
         ));
 
         assert!(matches!(
-            calc_final_ranking_per_review(&gen_dummy_rankings("".into(), 4, 2, 5, RandomIterator)),
-            ReviewRanking::Good
+            calc_final_ranking_with_confidence_per_review(&gen_dummy_rankings("".into(), 4, 2, 5, RandomIterator)),
+            FinalRankingWithConfidence {
+                review_ranking: Good,
+                confidence
+            } if confidence == (dec!(6) / dec!(11))
+
         ));
 
         assert!(matches!(
-            calc_final_ranking_per_review(&gen_dummy_rankings("".into(), 4, 1, 5, RandomIterator)),
-            ReviewRanking::FilteredOut
+            calc_final_ranking_with_confidence_per_review(&gen_dummy_rankings("".into(), 4, 1, 5, RandomIterator)),
+            FinalRankingWithConfidence {
+                review_ranking: FilteredOut,
+                confidence,
+            } if confidence == (dec!(5) / dec!(10))
         ));
 
         assert!(matches!(
-            calc_final_ranking_per_review(&gen_dummy_rankings("".into(), 3, 1, 1, RandomIterator)),
-            ReviewRanking::Excellent
+            calc_final_ranking_with_confidence_per_review(&gen_dummy_rankings("".into(), 3, 1, 1, RandomIterator)),
+            FinalRankingWithConfidence {
+                review_ranking: Excellent,
+                confidence,
+            } if confidence == (dec!(4) / dec!(5))
         ));
     }
 
@@ -231,6 +283,7 @@ mod tests {
                 .into_iter()
                 .zip(REPUTATION_DISAGREEMENT_MODIFIERS.into_iter())
                 .collect(),
+            SIMPLE_MAJORITY_CONFIDENCE,
         );
         assert!(results.get(VCA_1).is_none());
         let res = results.get(VCA_2).unwrap();
@@ -260,6 +313,7 @@ mod tests {
                 .into_iter()
                 .zip(REPUTATION_DISAGREEMENT_MODIFIERS.into_iter())
                 .collect(),
+            SIMPLE_MAJORITY_CONFIDENCE,
         );
         let res1 = results.get(VCA_1).unwrap();
         assert_eq!(res1.reputation, 1);
@@ -283,21 +337,55 @@ mod tests {
             (Rewards::new(8, 1), Rewards::ONE, Rewards::ONE),
             (Rewards::new(9, 1), Rewards::new(125, 2), Rewards::ONE),
         ];
-        for (agreement, reward_modifier, reputation_modifier) in inputs {
+        for (vca3_agreement, reward_modifier, reputation_modifier) in inputs {
             let rankings = (0..100)
                 .flat_map(|i| {
                     let vcas =
                         vec![VCA_1.to_owned(), VCA_2.to_owned(), VCA_3.to_owned()].into_iter();
-                    let (good, filtered_out) = if Rewards::from(i) < agreement * Rewards::from(100)
-                    {
-                        (3, 0)
-                    } else {
-                        (2, 1)
-                    };
+                    let (good, filtered_out) =
+                        if Rewards::from(i) < vca3_agreement * Rewards::from(100) {
+                            (3, 0)
+                        } else {
+                            (2, 1)
+                        };
                     gen_dummy_rankings(i.to_string(), 0, good, filtered_out, vcas).into_iter()
                 })
                 .collect::<Vec<_>>();
-            let results = calculate_veteran_advisors_incentives(
+            let results_simple_confidence = calculate_veteran_advisors_incentives(
+                &rankings,
+                total_rewards,
+                1..=200,
+                1..=200,
+                THRESHOLDS
+                    .into_iter()
+                    .zip(REWARDS_DISAGREEMENT_MODIFIERS.into_iter())
+                    .collect(),
+                THRESHOLDS
+                    .into_iter()
+                    .zip(REPUTATION_DISAGREEMENT_MODIFIERS.into_iter())
+                    .collect(),
+                SIMPLE_MAJORITY_CONFIDENCE,
+            );
+            let vca3_expected_reward_portion_simple_confidence =
+                vca3_agreement * Rewards::from(100) * reward_modifier;
+            dbg!(vca3_expected_reward_portion_simple_confidence);
+            dbg!(vca3_agreement, reward_modifier, reputation_modifier);
+            let vca3_expected_rewards_simple_confidence = total_rewards
+                / (Rewards::from(125 * 2) + vca3_expected_reward_portion_simple_confidence)
+                * vca3_expected_reward_portion_simple_confidence;
+            let res_vca3_simple_confidence = results_simple_confidence.get(VCA_3).unwrap();
+            assert_eq!(
+                res_vca3_simple_confidence.reputation,
+                (Rewards::from(100) * vca3_agreement * reputation_modifier)
+                    .to_u64()
+                    .unwrap()
+            );
+            assert!(are_close(
+                res_vca3_simple_confidence.rewards,
+                vca3_expected_rewards_simple_confidence
+            ));
+
+            let results_qualified_confidence = calculate_veteran_advisors_incentives(
                 &rankings,
                 total_rewards,
                 1..=200,
@@ -310,21 +398,29 @@ mod tests {
                     .into_iter()
                     .zip(REPUTATION_DISAGREEMENT_MODIFIERS.into_iter())
                     .collect(),
+                QUALIFIED_MAJORITY_CONFIDENCE,
             );
-            let expected_reward_portion = agreement * Rewards::from(100) * reward_modifier;
-            dbg!(expected_reward_portion);
-            dbg!(agreement, reward_modifier, reputation_modifier);
-            let expected_rewards = total_rewards
-                / (Rewards::from(125 * 2) + expected_reward_portion)
-                * expected_reward_portion;
-            let res = results.get(VCA_3).unwrap();
+
+            let vca3_expected_reward_portion_qualified_confidence = Rewards::from(100) * dec!(1.25); // low confidence so max reward modifier, agreement ratio doesn't count as all and rankings are all eligible
+            dbg!(vca3_expected_reward_portion_qualified_confidence);
+            dbg!(vca3_agreement, reward_modifier, reputation_modifier);
+
+            let vca3_expected_rewards_qualified_confidence = total_rewards
+                / (Rewards::from(125 * 2) + vca3_expected_reward_portion_qualified_confidence)
+                * vca3_expected_reward_portion_qualified_confidence; // 1/3 of the reward
+
+            let res_vca3_qualified_confidence = results_qualified_confidence.get(VCA_3).unwrap();
+
             assert_eq!(
-                res.reputation,
-                (Rewards::from(100) * agreement * reputation_modifier)
+                res_vca3_qualified_confidence.reputation,
+                (Rewards::from(100)) // all assessment are valid since confidence is low (2/3 < 0.7)
                     .to_u64()
                     .unwrap()
             );
-            assert!(are_close(res.rewards, expected_rewards));
+            assert!(are_close(
+                res_vca3_qualified_confidence.rewards,
+                vca3_expected_rewards_qualified_confidence
+            ));
         }
     }
 }