Avoid blocking in the PartitionProcessorManager when being in the control loop

Summary:
- Improve tracing of ControlProcessors commands
- Run the spawn of partition processors concurrently

Author: muhamadazmy

crates/core/src/metadata/mod.rs

@@ -18,6 +18,7 @@ use std::sync::{Arc, OnceLock};
 use arc_swap::{ArcSwap, AsRaw};
 use enum_map::EnumMap;
 use tokio::sync::{mpsc, oneshot, watch};
+use tracing::instrument;
 
 use restate_types::live::{Live, Pinned};
 use restate_types::logs::metadata::Logs;
@@ -175,6 +176,7 @@ impl Metadata {
     }
 
     /// Returns when the metadata kind is at the provided version (or newer)
+    #[instrument(level = "debug", skip(self))]
     pub async fn wait_for_version(
         &self,
         metadata_kind: MetadataKind,

crates/types/src/cluster/cluster_state.rs

@@ -97,7 +97,9 @@ pub struct DeadNode {
     pub last_seen_alive: Option<MillisSinceEpoch>,
 }
 
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, Eq, PartialEq, IntoProto)]
+#[derive(
+    Debug, Clone, Copy, Serialize, Deserialize, Eq, PartialEq, IntoProto, derive_more::Display,
+)]
 #[proto(target = "crate::protobuf::cluster::RunMode")]
 pub enum RunMode {
     Leader,