Skip to content

Commit

Permalink
fix(compute): fix compute standalone memory calculate
Browse files Browse the repository at this point in the history
  • Loading branch information
Li0k committed Nov 1, 2024
1 parent e5dfe57 commit ce291da
Show file tree
Hide file tree
Showing 4 changed files with 23 additions and 5 deletions.
2 changes: 1 addition & 1 deletion src/cmd_all/src/standalone.rs
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ pub async fn standalone(
let compute = if let Some(opts) = compute_opts {
tracing::info!("starting compute-node thread with cli args: {:?}", opts);
let service = Service::spawn("compute", |shutdown| {
risingwave_compute::start(opts, shutdown)
risingwave_compute::start_standalone(opts, shutdown)
});
Some(service)
} else {
Expand Down
17 changes: 16 additions & 1 deletion src/compute/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -206,10 +206,25 @@ fn validate_opts(opts: &ComputeNodeOpts) {

use crate::server::compute_node_serve;

/// Start compute node
pub fn start(
opts: ComputeNodeOpts,
shutdown: CancellationToken,
) -> Pin<Box<dyn Future<Output = ()> + Send>> {
start_impl(opts, false, shutdown)
}

pub fn start_standalone(
opts: ComputeNodeOpts,
shutdown: CancellationToken,
) -> Pin<Box<dyn Future<Output = ()> + Send>> {
start_impl(opts, true, shutdown)
}

/// Start compute node
fn start_impl(
opts: ComputeNodeOpts,
is_standalone: bool,
shutdown: CancellationToken,
) -> Pin<Box<dyn Future<Output = ()> + Send>> {
// WARNING: don't change the function signature. Making it `async fn` will cause
// slow compile in release mode.
Expand Down
5 changes: 3 additions & 2 deletions src/compute/src/memory/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,12 @@ fn gradient_reserve_memory_bytes(total_memory_bytes: usize) -> usize {
/// limits are calculated based on the proportions to total `non_reserved_memory_bytes`.
pub fn extract_hummock_memory_config(
non_reserved_memory_bytes: usize,
embedded_compactor_enabled: bool,
is_compactor_hybird_deplyment: bool,
storage_config: &StorageConfig,
is_serving: bool,
) -> (StorageMemoryConfig, Option<CompactorMemoryConfig>) {
let (storage_memory_proportion, compactor_memory_proportion) = if embedded_compactor_enabled {
let (storage_memory_proportion, compactor_memory_proportion) = if is_compactor_hybird_deplyment
{
(STORAGE_MEMORY_PROPORTION, COMPACTOR_MEMORY_PROPORTION)
} else {
(STORAGE_MEMORY_PROPORTION + COMPACTOR_MEMORY_PROPORTION, 0.0)
Expand Down
4 changes: 3 additions & 1 deletion src/compute/src/server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ pub async fn compute_node_serve(
listen_addr: SocketAddr,
advertise_addr: HostAddr,
opts: ComputeNodeOpts,
is_standalone: bool,
shutdown: CancellationToken,
) {
// Load the configuration.
Expand Down Expand Up @@ -138,11 +139,12 @@ pub async fn compute_node_serve(

let embedded_compactor_enabled =
embedded_compactor_enabled(state_store_url, config.storage.disable_remote_compactor);
let is_compactor_hybrid_deployment = embedded_compactor_enabled || is_standalone;

let (reserved_memory_bytes, non_reserved_memory_bytes) = reserve_memory_bytes(&opts);
let (storage_memory_config, compactor_memory_config) = extract_hummock_memory_config(
non_reserved_memory_bytes,
embedded_compactor_enabled,
is_compactor_hybrid_deployment,
&config.storage,
!opts.role.for_streaming(),
);
Expand Down

0 comments on commit ce291da

Please sign in to comment.