Remove refute v3 (into dev)

package.json

@@ -52,9 +52,9 @@
   "dependencies": {
     "@hapi/sntp": "3.1.1",
     "@mapbox/node-pre-gyp": "1.0.10",
-    "@shardus/crypto-utils": "git+https://github.com/shardeum/lib-crypto-utils#dev",
-    "@shardus/net": "git+https://github.com/shardeum/lib-net#dev",
-    "@shardus/types": "git+https://github.com/shardeum/lib-types#dev",
+    "@shardus/crypto-utils": "git+https://github.com/shardeum/lib-crypto-utils#itn4",
+    "@shardus/net": "git+https://github.com/shardeum/lib-net#itn4",
+    "@shardus/types": "git+https://github.com/shardeum/lib-types#itn4",
     "@types/better-sqlite3": "7.6.3",
     "body-parser": "1.18.3",
     "bytenode": "1.3.4",

src/config/deprecated/server.json

@@ -39,6 +39,10 @@
     "maxJoinedPerCycle": 1,
     "maxSyncingPerCycle": 5,
     "maxRotatedPerCycle": 1,
+    "maxProblematicNodeRemovalsPerCycle": 1,
+    "problematicNodeConsecutiveRefuteThreshold": 6,
+    "problematicNodeRefutePercentageThreshold": 0.1,
+    "problematicNodeHistoryLength": 100,
     "firstCycleJoin": 10,
     "maxPercentOfDelta": 40,
     "minScaleReqsNeeded": 5,

src/config/server.ts

@@ -69,6 +69,15 @@ const SERVER_CONFIG: StrictServerConfiguration = {
     maxSyncTimeFloor: 1200,
     maxNodeForSyncTime: 9,
     maxRotatedPerCycle: 1,
+    flexibleRotationDelta: 1,
+    flexibleRotationEnabled: false,
+    enableProblematicNodeRemoval: false,
+    enableProblematicNodeRemovalOnCycle: 20000,
+    maxProblematicNodeRemovalsPerCycle: 1,
+    problematicNodeConsecutiveRefuteThreshold: 6,
+    problematicNodeRefutePercentageThreshold: 0.1,
+    problematicNodeHistoryLength: 100,
+    problematicNodeRemovalCycleFrequency: 5,
     firstCycleJoin: 10,
     maxPercentOfDelta: 40,
     minScaleReqsNeeded: 5,

src/debug/debug.ts

@@ -6,6 +6,10 @@ import Trie from 'trie-prefix-tree'
 import { isDebugModeMiddleware, isDebugModeMiddlewareMedium } from '../network/debugMiddleware'
 import { nestedCountersInstance } from '../utils/nestedCounters'
 import { logFlags } from '../logger'
+import { currentCycle } from '../p2p/CycleCreator'
+import * as ProblemNodeHandler from '../p2p/ProblemNodeHandler'
+import { Node } from '@shardus/types/build/src/p2p/NodeListTypes'
+import { nodes } from '../p2p/NodeList'
 const tar = require('tar-fs')
 const fs = require('fs')
 
@@ -141,6 +145,30 @@ class Debug {
       res.json({ success: true })
       return
     })
+    this.network.registerExternalGet('debug_problemNodeTrackerDump', isDebugModeMiddleware, (_, res) => {
+      try {
+        const dump: Record<string, any> = {}
+
+        // Collect data for all nodes that have any refute history
+        for (const [nodeId, node] of nodes as Map<string, Node>) {
+          if (node.refuteCycles?.size > 0) {
+            const refuteCycles = Array.from(node.refuteCycles).sort((a, b) => a - b)
+            dump[nodeId] = {
+              refuteCycles,
+              stats: {
+                refutePercentage: ProblemNodeHandler.getRefutePercentage(node.refuteCycles, currentCycle),
+                consecutiveRefutes: ProblemNodeHandler.getConsecutiveRefutes(refuteCycles, currentCycle),
+                isProblematic: ProblemNodeHandler.isNodeProblematic(node, currentCycle)
+              }
+            }
+          }
+        }
+
+        res.json({ success: true, data: { nodeHistories: dump } })
+      } catch (e) {
+        res.json({ success: false, error: e.message })
+      }
+    })
   }
 }
 

src/p2p/Join/routes.ts

@@ -254,22 +254,22 @@ const unjoinRoute: P2P.P2PTypes.Route<Handler> = {
   method: 'POST',
   name: 'unjoin',
   handler: (req, res) => {
-    const unjoinRequest = req.body
-    const processResult = processNewUnjoinRequest(unjoinRequest)
-    if (processResult.isErr()) {
-      res.status(500).json({ error: processResult.error })
-      return
-    }
-
-    // we need to remove the unjoin request this cycle since we are waiting until next cycle to gossip it to the network
-    // processNewUnjoinRequest automatically adds the unjoin request to newUnjoinRequests it will be added back to our list
-    // once we process the queued request next cycle
-    removeUnjoinRequest(unjoinRequest.publicKey)
-
-    queueUnjoinRequest(unjoinRequest)
-
-    res.status(200).json()
-    return
+   try {
+     const unjoinRequest = req.body
+
+     const processResult = processNewUnjoinRequest(unjoinRequest)
+     if (processResult.isErr()) {
+       res.status(500).json({ error: processResult.error.message })
+       return
+     }
+
+     removeUnjoinRequest(unjoinRequest.publicKey)
+     queueUnjoinRequest(unjoinRequest)
+
+     res.status(200).json({ message: 'Unjoin request processed successfully' })
+   } catch (error) {
+     res.status(500).json({ error: error.message || 'An unknown error occurred' })
+   }
   },
 }
 

src/p2p/Join/v2/unjoin.ts

@@ -1,6 +1,6 @@
 import { crypto } from '../../Context'
 import { err, ok, Result } from 'neverthrow'
-import { hexstring } from '@shardus/types'
+import { hexstring, P2P } from '@shardus/types'
 import * as utils from '../../../utils'
 import * as http from '../../../http'
 import * as NodeList from '../../NodeList'
@@ -9,6 +9,7 @@ import { getActiveNodesFromArchiver, getRandomAvailableArchiver } from '../../Ut
 import { logFlags } from '../../../logger'
 import * as CycleChain from '../../CycleChain'
 import { SignedUnjoinRequest } from '@shardus/types/build/src/p2p/JoinTypes'
+import { getPublicNodeInfo } from '../../Self'
 
 /** A Set of new public keys of nodes that have submitted unjoin requests. */
 const newUnjoinRequests: Set<SignedUnjoinRequest> = new Set()
@@ -19,20 +20,10 @@ const newUnjoinRequests: Set<SignedUnjoinRequest> = new Set()
 export async function submitUnjoin(): Promise<Result<void, Error>> {
   const publicKey = crypto.keypair.publicKey
 
-  const foundInStandbyNodes = getStandbyNodesInfoMap().has(publicKey)
-  if (!foundInStandbyNodes) {
+  if (getPublicNodeInfo(true).status !== P2P.P2PTypes.NodeStatus.STANDBY) {
     return err(new Error('node is not in standby. Do not send unjoin request'))
   }
 
-  if(!CycleChain.getNewest()) {
-    return err(new Error('No cycle chain found. Do not send unjoin request'))
-  }
-
-  const unjoinRequest = crypto.sign({
-    publicKey: publicKey,
-    cycleNumber: CycleChain.getNewest().counter,
-  })
-
   const archiver = getRandomAvailableArchiver()
   try {
     const activeNodesResult = await getActiveNodesFromArchiver(archiver)
@@ -47,6 +38,19 @@ export async function submitUnjoin(): Promise<Result<void, Error>> {
     // not being properly removed isn't that big a deal. Standby refresh will take care of them anyways.
     // If we really want to solve this, can do so by sending request to numRotatedOut + 1 nodes.
     const node = utils.getRandom(activeNodes.nodeList, 1)[0]
+    const cycleRecord: P2P.CycleCreatorTypes.CycleRecord = await http.get(
+      `${node.ip}:${node.port}/newest-cycle-record`
+    )
+
+    if (!cycleRecord?.counter) {
+      return err(new Error('No cycle counter found. Do not send unjoin request'))
+    }
+
+    const unjoinRequest = crypto.sign({
+      publicKey: publicKey,
+      cycleNumber: cycleRecord.counter,
+    })
+
     await http.post(`${node.ip}:${node.port}/unjoin`, unjoinRequest)
     return ok(void 0)
   } catch (e) {