Merge branch 'main' into formal-spec

Author: bwbush

Logbook.md

@@ -1,3 +1,71 @@
+# Leios logbook
+
+## 2024-09-20
+
+### Team meeting
+
+- Introductions and roles
+    - Excellent balance of Agda, Haskell, and Rust skills
+- Reviewed administrative informoration
+    - PI7 Jira tickets, mirrored at [#16](https://github.com/input-output-hk/ouroboros-leios/issues/16), [#17](https://github.com/input-output-hk/ouroboros-leios/issues/17), [#18](https://github.com/input-output-hk/ouroboros-leios/issues/18)
+    - Nominal objectives, tasks, and deliverables for next 12 months
+- Work agreement
+    - Write down “everything” in a “research journal”
+        - what we do
+        - why we do it
+        - what are the expected results vs. actual results.
+    - Regular (at least weekly) technical review of all work done by everyone
+    - Show & tell sessions
+    - Communicate with the stakeholders (including community) on a regular basis
+    - Experimenting, pivoting, and dead-ends provide valuable learnings and should be documented in logbook
+    - Processes and workflows can emerge from our needs, and do not have to match typical production enviroments
+        - However, QA and checking each others' work is important
+    - Ensure all results are “easily” reproducible by the community
+    - Arnaud will pair with engineering staff each week in October
+    - Technical report each quarter -- the next will be in December
+    - CIP at conclusion of innovation workstream
+- Project resources listed at [discussion #15](https://github.com/input-output-hk/ouroboros-leios/discussions/15)
+- Open discussion
+    - Leios flavors
+        - Blob, short, simplified, and full
+        - All flavors have the same security guarantees
+        - Blob Leios does not deal with Cardano transactions
+        - Full Leios adds robust throughput even in the face of performance issues in base protocol
+    - Details of interaction with ledger
+        - Fairness for fees
+        - Wastage (i.e., duplicate or conflicting transactions)
+        - Time-dependent smart contracts
+    - Demo of initial DeltaQ rust implementation by Roland
+    - Brian will coordinate with David about contracts and with Hans about tweeting
+- Communication and collaboration
+    - Use slack channel for admistrative and scheduling
+    - Use github [discussions](https://github.com/input-output-hk/ouroboros-leios/discussions) or [pull requests](https://github.com/input-output-hk/ouroboros-leios/pulls)
+        - When relevant, summarize discussion results or PR comments in [this logbook](./Logbook.md)
+    - Put experimental code and work in progress on the main branch of [this repository](./)
+        - Avoid long-lived branches
+        - If warranted, code can later be separated into a new repository via `git subtree`
+        - Mostly keep main branch passing the CI, but short-lived exceptions are okay
+        - Describe new work and findings in [this logbook](./Logbook.md)
+    - Approximately three working meetings per week
+        - Each meeting (or slack) will settle the topic for the next meeting
+- Next steps
+    - Meetings next week approximately at 1400 UTC
+        - Duncan will present on existing ouroboros network
+        - Two meetings to discuss, brainstorm, and plan network simulations
+    - Future meetings
+        - Duncan will present on ouroboros consensus protocol
+        - Duncan will present on praos lessons learned
+        - Arnaud, Yves, or Brian will present on peras lessons learned
+        - Andre will present on formalization lessons learned
+        - Additional protocol and formalization meetings
+        - At some point we'll meet with PNSol about DeltaQ collaboration
+    - Possible in-person meeting mid December or January?
+    - Work focus
+        - Roland and Yves will collaborate on DeltaQ
+        - Everyone should familiarize themselves with the [simulation/](simulation/) and [leios-sim/](leios-sim/) code
+            - Live version of `leios-sim` at https://leios-simulation.cardano-scaling.org/index.html
+            - Run `simulation` locally via [instructions below](#running-ouroborous-net-viz-in-the-browser)
+
 ## 2024-08-30
 
 ### Nix and CI support for Leios specification

leios-sim/index.js

@@ -73,6 +73,9 @@ document.addEventListener('DOMContentLoaded', () => {
   // queue.
   var queuedIBs = [];
 
+  // We keep track of recently endorsed IBs to avoid double counting
+  var recently_endorsed_IB_UUIDs = [];
+
   // accumulate the size of all endorsed IBs.
   var total_endorsed_data = 0;
   var total_dropped_data = 0;
@@ -133,12 +136,15 @@ document.addEventListener('DOMContentLoaded', () => {
           break;
         }
         case 'ReceivedEB': {
-
-          if (logData.receivedEB.eb_linked_IBs.length != 0) {
-            queuedIBs = _.differenceWith(queuedIBs, logData.receivedEB.eb_linked_IBs, _.isEqual);
-            total_endorsed_data += logData.receivedEB.eb_linked_IBs.map(ib => ib.ib_size).reduce((a, b) => a + b, 0);
-            total_endorsed_IBs += logData.receivedEB.eb_linked_IBs.length;
-            total_endorsed_latency += logData.receivedEB.eb_linked_IBs.map(ib => logData.receivedEB.eb_slot - ib.ib_slot).reduce((a, b) => a + b, 0);
+          var new_endorsed_IBs = logData.receivedEB.eb_linked_IBs.filter(ib => !recently_endorsed_IB_UUIDs.includes(ib.ib_UUID));
+
+          if (new_endorsed_IBs.length != 0) {
+            queuedIBs = _.differenceWith(queuedIBs, new_endorsed_IBs, _.isEqual);
+            total_endorsed_data += new_endorsed_IBs.map(ib => ib.ib_size).reduce((a, b) => a + b, 0);
+            total_endorsed_IBs += new_endorsed_IBs.length;
+            total_endorsed_latency += new_endorsed_IBs.map(ib => logData.receivedEB.eb_slot - ib.ib_slot).reduce((a, b) => a + b, 0);
+            // TODO find a reasonable lenght to truncate to
+            recently_endorsed_IB_UUIDs = recently_endorsed_IB_UUIDs.concat(new_endorsed_IBs.map(ib => ib.ib_UUID)).slice(-500);
           }
 
           // We know that future EBs will link IBs whose slots are

leios-sim/leios-sim.cabal

@@ -43,6 +43,7 @@ library
     , http-types
     , text
     , time
+    , uuid
     , wai
     , wai-extra
     , wai-middleware-static

leios-sim/src/Leios/Model.hs

@@ -38,11 +38,12 @@ module Leios.Model where
 
 import Prelude hiding (init)
 
+import Data.UUID (UUID)
 import Control.Applicative (asum)
 import Control.Concurrent.Class.MonadSTM.TChan (TChan, newTChanIO, readTChan, writeTChan)
 import Control.Concurrent.Class.MonadSTM.TQueue (TQueue, newTQueueIO, readTQueue)
 import Control.Concurrent.Class.MonadSTM.TVar (TVar, check, modifyTVar', newTVarIO, readTVar, readTVarIO, writeTVar)
-import Control.Monad (forever, replicateM, replicateM_)
+import Control.Monad (foldM, forever, replicateM, replicateM_)
 import Control.Monad.Class.MonadAsync (Async, Concurrently (Concurrently, runConcurrently), MonadAsync, async, race_)
 import Control.Monad.Class.MonadSTM (MonadSTM, STM, atomically, retry)
 import Control.Monad.Class.MonadTimer (MonadDelay, MonadTimer, threadDelay)
@@ -60,7 +61,7 @@ import qualified Data.PQueue.Max as PQueue
 import Data.Word (Word64)
 import GHC.Generics (Generic)
 import Leios.Trace (mkTracer)
-import System.Random (RandomGen, mkStdGen, randomR, split)
+import System.Random (RandomGen, mkStdGen, random, randomR, split)
 import Text.Pretty.Simple (pPrint)
 
 --------------------------------------------------------------------------------
@@ -71,16 +72,16 @@ import Text.Pretty.Simple (pPrint)
 --
 -- TODO: we might want to split this between constants and parameters that can be tweaked at runtime.
 data Parameters = Parameters
-  { _L :: NumberOfSlots
+  { _L            :: NumberOfSlots
   -- ^  Slice length.
-  , λ :: NumberOfSlices
+  , λ             :: NumberOfSlices
   -- ^ Number of slices (of size '_L') the diffusion period takes.
   , nodeBandwidth :: BitsPerSecond
-  , ibSize :: NumberOfBits
+  , ibSize        :: NumberOfBits
   -- ^ Size of the diffusion block.
-  , f_I :: IBFrequency
-  , f_E :: EBFrequency
-  , initialSeed :: Int
+  , f_I           :: IBFrequency
+  , f_E           :: EBFrequency
+  , initialSeed   :: Int
   }
   deriving (Show, Generic)
   deriving anyclass (Aeson.ToJSON, Aeson.FromJSON)
@@ -139,6 +140,7 @@ data IB = IB
   { ib_slot :: Slot
   , ib_producer :: NodeId
   , ib_size :: NumberOfBits
+  , ib_UUID :: UUID
   }
   deriving (Show, Eq, Generic)
   deriving anyclass (Aeson.ToJSON, Aeson.FromJSON)
@@ -268,13 +270,16 @@ node nodeId nodeStakePercent initialGenerator tracer world continueTVar = do
           -- Generate IB blocks
           let (numberOfIBsInThisSlot, generator1) =
                 slotsLed generator (getIBFrequency f_I)
-          replicateM_ numberOfIBsInThisSlot $ produceIB slot
+          generator2 <- foldM
+            produceIB
+            generator1
+            $ replicate numberOfIBsInThisSlot slot
           -- Generate EB blocks
-          let (numberOfEBsInThisSlot, generator2) =
-                slotsLed generator1 (getEBFrequency f_E)
+          let (numberOfEBsInThisSlot, generator3) =
+                slotsLed generator2 (getEBFrequency f_E)
           replicateM_ numberOfEBsInThisSlot $ produceEB slot
           -- Continue
-          loop generator2
+          loop generator3
     loop initialGenerator
 
   consumer = forever $ do
@@ -293,14 +298,22 @@ node nodeId nodeStakePercent initialGenerator tracer world continueTVar = do
       q = ceiling f
       (nodeLeads, nextGenerator) =
         leadsMultiple generator q nodeStakePercent f
-     in
+      in
       (length $ filter id nodeLeads, nextGenerator)
 
-  produceIB slot = do
+  produceIB generator slot = do
     Parameters{ibSize} <- getParams world
-    let newIB = IB{ib_slot = slot, ib_producer = nodeId, ib_size = ibSize}
+    let
+      (uuid,nextGenerator) = random generator
+      newIB = IB {
+        ib_slot = slot
+        , ib_producer = nodeId
+        , ib_size = ibSize
+        , ib_UUID = uuid
+        }
     traceWith tracer (ProducedIB newIB)
     MsgIB newIB `sendTo` world
+    pure nextGenerator
 
   produceEB slot = do
     Parameters{_L, λ} <- getParams world