Restart compute node and temporal join

[silentsokolov]

Describe the bug

When using a temporary join, a node restart can trigger an error that prevents the node from being started again.

Error message/log

Example

   2024-12-13T11:33:22.505022841Z  INFO risingwave_stream::task::barrier_manager::managed_state: ignored syncing data for the first barrier prev_epoch=7658157758808064

 thread 'rw-streaming' panicked at src/stream/src/executor/temporal_join.rs:97:13:

 key OwnedRow([Some(Serial(Serial(490115337925517313))), Some(Int64(35169))]) should be in the cache

 stack backtrace:

    0:     0x5608c414c1d9 - std::backtrace_rs::backtrace::libunwind::trace::h81f95f911fafa2e0

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/../../backtrace/src/backtrace/libunwind.rs:116:5

    1:     0x5608c414c1d9 - std::backtrace_rs::backtrace::trace_unsynchronized::h2e88326ce80cfb26

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5

    2:     0x5608c414c1d9 - std::sys::backtrace::_print_fmt::h3318b90cf1bd0c10

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/sys/backtrace.rs:65:5

    3:     0x5608c414c1d9 - <std::sys::backtrace::BacktraceLock::print::DisplayBacktrace as core::fmt::Display>::fmt::hfb260f93257b2f2e

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/sys/backtrace.rs:40:26

    4:     0x5608b582ee5b - core::fmt::rt::Argument::fmt::h78c29ca6c560b840

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/fmt/rt.rs:173:76

    5:     0x5608b582ee5b - core::fmt::write::h70fe4701d8d7a171

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/fmt/mod.rs:1182:21

    6:     0x5608c41470bf - std::io::Write::write_fmt::h14783b7a2197dfce

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/io/mod.rs:1827:15

    7:     0x5608c414d301 - std::sys::backtrace::BacktraceLock::print::hc4ed4e400debb2bf

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/sys/backtrace.rs:43:9

    8:     0x5608c414d301 - std::panicking::default_hook::{{closure}}::hfff4b120f05a97bf

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:269:22

    9:     0x5608c414cfe7 - std::panicking::default_hook::h52349d4986dd35a0

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:296:9

   10:     0x5608c0f8311b - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::he9972cd62e1b6c03

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/alloc/src/boxed.rs:2084:9

   11:     0x5608c0f8311b - risingwave_rt::panic_hook::set_panic_hook::{{closure}}::hb4d8d816619ba0af

                                at /risingwave/src/utils/runtime/src/panic_hook.rs:25:9

   12:     0x5608c0f8311b - std::panicking::update_hook::{{closure}}::h1b85f23f2996e971

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:233:47

   13:     0x5608c414db6f - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::h3da72443baa8e06b

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/alloc/src/boxed.rs:2084:9

   14:     0x5608c414db6f - std::panicking::rust_panic_with_hook::hb85b5d3fee996073

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:808:13

   15:     0x5608c414d857 - std::panicking::begin_panic_handler::{{closure}}::hea92f85bb1e00425

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:674:13

   16:     0x5608c414c699 - std::sys::backtrace::__rust_end_short_backtrace::h4835e8f15136c715

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/sys/backtrace.rs:168:18

   17:     0x5608c414d4d3 - rust_begin_unwind

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:665:5

   18:     0x5608b582b3e2 - core::panicking::panic_fmt::hb0937d4ccfd82515

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/panicking.rs:74:14

   19:     0x5608c231d5c8 - risingwave_stream::executor::temporal_join::JoinEntry::remove::hf82f799c404dcf9a

                                at /risingwave/src/stream/src/executor/temporal_join.rs:97:13

   20:     0x5608c231d5c8 - risingwave_stream::executor::temporal_join::TemporalSide<K,S>::update::h2b7e834bea031c2a

                                at /risingwave/src/stream/src/executor/temporal_join.rs:194:58

   21:     0x5608c217327c - risingwave_stream::executor::temporal_join::TemporalJoinExecutor<K,S,_,_>::into_stream::{{closure}}::h981916f4fc949d9a

                                at /risingwave/src/stream/src/executor/temporal_join.rs:833:21

   22:     0x5608c217327c - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h9b75b85fd76a356f

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   23:     0x5608c1d6132a - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h89379077e6150140

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   24:     0x5608c1d6132a - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h38396531d651abe7

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   25:     0x5608c1d6132a - futures_util::stream::stream::StreamExt::poll_next_unpin::h154f4b34eab6b1fc

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9

   26:     0x5608c1d6132a - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h55b7892ee7493236

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9

   27:     0x5608c1d6132a - <await_tree::future::Instrumented<F,_> as core::future::future::Future>::poll::he063ee30d8e33e62

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/await-tree-0.2.1/src/future.rs:119:15

   28:     0x5608c34c2cef - risingwave_stream::executor::wrapper::trace::instrument_await_tree::{{closure}}::h65df7c64fe49803e

                                at /risingwave/src/stream/src/executor/wrapper/trace.rs:121:10

   29:     0x5608c34c2cef - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h4012cf30c241de43

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   30:     0x5608c34c2cef - risingwave_stream::executor::wrapper::schema_check::schema_check::{{closure}}::hcdaef53dfdb1c3b6

                                at /risingwave/src/stream/src/executor/wrapper/schema_check.rs:24:1

   31:     0x5608c34c2cef - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h8378a1d556b63d1a

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   32:     0x5608c34c2cef - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::hbced3fa0d3159b50

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   33:     0x5608c34c2cef - futures_util::stream::stream::StreamExt::poll_next_unpin::h2b17a56f54acd6a8

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9

   34:     0x5608c34bda8c - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h7f0ffc044dac8996

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9

   35:     0x5608c34bda8c - risingwave_stream::executor::wrapper::epoch_check::epoch_check::{{closure}}::h98262a88b11c76e7

                                at /risingwave/src/stream/src/executor/wrapper/epoch_check.rs:31:44

   36:     0x5608c34bda8c - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::hfcf54f81b20696f2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   37:     0x5608c34bda8c - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::hf0f90fee65c4d60a

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   38:     0x5608c34bda8c - <S as futures_core::stream::TryStream>::try_poll_next::h9ffa6315b1cc546a

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:196:9

   39:     0x5608c34bda8c - futures_util::stream::try_stream::TryStreamExt::try_poll_next_unpin::he56f26f8a8be87ab

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/try_stream/mod.rs:1131:9

   40:     0x5608c1e21cc8 - <futures_util::stream::try_stream::try_next::TryNext<St> as core::future::future::Future>::poll::h74746373b1039b83

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/try_stream/try_next.rs:32:9

   41:     0x5608c1e21cc8 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h0e1e4bd427b12e86

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

   42:     0x5608c1e21cc8 - tokio::task::task_local::LocalKey<T>::scope_inner::h7edb0552146c04ac

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

   43:     0x5608c1e21cc8 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::hd7d34eb12d8caab2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

   44:     0x5608c34c381c - risingwave_common::util::epoch::task_local::scope::{{closure}}::hab192e60dda84fbb

                                at /risingwave/src/common/src/util/epoch.rs:249:47

   45:     0x5608c34c381c - risingwave_stream::executor::wrapper::epoch_provide::epoch_provide::{{closure}}::h7ab6553c8c5b1b7b

                                at /risingwave/src/stream/src/executor/wrapper/epoch_provide.rs:31:59

   46:     0x5608c34c381c - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h5a83fafe411e9554

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   47:     0x5608c34c381c - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h10d754f96fe381ec

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   48:     0x5608c34c381c - futures_util::stream::stream::StreamExt::poll_next_unpin::h4401b57aaca2d4f2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9

   49:     0x5608c32ec6cc - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h1c1f040471113424

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9

   50:     0x5608c32ec6cc - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h369eb8c15e0a31ea

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9

   51:     0x5608c1e2c28a - risingwave_stream::executor::wrapper::trace::trace::{{closure}}::h5a79a0a254b17b9e

                                at /risingwave/src/stream/src/executor/wrapper/trace.rs:58:69

   52:     0x5608c1e2c28a - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h4c5f95930c499f9c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   53:     0x5608c35542d1 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h19c2117822980a29

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   54:     0x5608c299d58f - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h89379077e6150140

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   55:     0x5608c299d58f - risingwave_stream::executor::filter::FilterExecutor::execute_inner::{{closure}}::he75eca12ddfe9559

                                at /risingwave/src/stream/src/executor/filter.rs:134:5

   56:     0x5608c299d58f - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h5a39b90262b30141

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   57:     0x5608c1d6132a - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h89379077e6150140

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   58:     0x5608c1d6132a - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h38396531d651abe7

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   59:     0x5608c1d6132a - futures_util::stream::stream::StreamExt::poll_next_unpin::h154f4b34eab6b1fc

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9

   60:     0x5608c1d6132a - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h55b7892ee7493236

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9

   61:     0x5608c1d6132a - <await_tree::future::Instrumented<F,_> as core::future::future::Future>::poll::he063ee30d8e33e62

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/await-tree-0.2.1/src/future.rs:119:15

   62:     0x5608c34c2cef - risingwave_stream::executor::wrapper::trace::instrument_await_tree::{{closure}}::h65df7c64fe49803e

                                at /risingwave/src/stream/src/executor/wrapper/trace.rs:121:10

   63:     0x5608c34c2cef - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h4012cf30c241de43

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   64:     0x5608c34c2cef - risingwave_stream::executor::wrapper::schema_check::schema_check::{{closure}}::hcdaef53dfdb1c3b6

                                at /risingwave/src/stream/src/executor/wrapper/schema_check.rs:24:1

   65:     0x5608c34c2cef - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h8378a1d556b63d1a

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   66:     0x5608c34c2cef - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::hbced3fa0d3159b50

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   67:     0x5608c34c2cef - futures_util::stream::stream::StreamExt::poll_next_unpin::h2b17a56f54acd6a8

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9

   68:     0x5608c34bda8c - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h7f0ffc044dac8996

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9

   69:     0x5608c34bda8c - risingwave_stream::executor::wrapper::epoch_check::epoch_check::{{closure}}::h98262a88b11c76e7

                                at /risingwave/src/stream/src/executor/wrapper/epoch_check.rs:31:44

   70:     0x5608c34bda8c - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::hfcf54f81b20696f2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   71:     0x5608c34bda8c - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::hf0f90fee65c4d60a

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   72:     0x5608c34bda8c - <S as futures_core::stream::TryStream>::try_poll_next::h9ffa6315b1cc546a

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:196:9

   73:     0x5608c34bda8c - futures_util::stream::try_stream::TryStreamExt::try_poll_next_unpin::he56f26f8a8be87ab

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/try_stream/mod.rs:1131:9

   74:     0x5608c1e21cc8 - <futures_util::stream::try_stream::try_next::TryNext<St> as core::future::future::Future>::poll::h74746373b1039b83

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/try_stream/try_next.rs:32:9

   75:     0x5608c1e21cc8 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h0e1e4bd427b12e86

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

   76:     0x5608c1e21cc8 - tokio::task::task_local::LocalKey<T>::scope_inner::h7edb0552146c04ac

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

   77:     0x5608c1e21cc8 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::hd7d34eb12d8caab2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

   78:     0x5608c34c381c - risingwave_common::util::epoch::task_local::scope::{{closure}}::hab192e60dda84fbb

                                at /risingwave/src/common/src/util/epoch.rs:249:47

   79:     0x5608c34c381c - risingwave_stream::executor::wrapper::epoch_provide::epoch_provide::{{closure}}::h7ab6553c8c5b1b7b

                                at /risingwave/src/stream/src/executor/wrapper/epoch_provide.rs:31:59

   80:     0x5608c34c381c - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h5a83fafe411e9554

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   81:     0x5608c34c381c - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h10d754f96fe381ec

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   82:     0x5608c34c381c - futures_util::stream::stream::StreamExt::poll_next_unpin::h4401b57aaca2d4f2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9

   83:     0x5608c32ec6cc - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h1c1f040471113424

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9

   84:     0x5608c32ec6cc - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h369eb8c15e0a31ea

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9

   85:     0x5608c1e2c28a - risingwave_stream::executor::wrapper::trace::trace::{{closure}}::h5a79a0a254b17b9e

                                at /risingwave/src/stream/src/executor/wrapper/trace.rs:58:69

   86:     0x5608c1e2c28a - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h4c5f95930c499f9c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   87:     0x5608c35542d1 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h19c2117822980a29

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   88:     0x5608c35536ea - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h89379077e6150140

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   89:     0x5608c35536ea - <risingwave_stream::executor::dispatch::DispatchExecutor as risingwave_stream::executor::StreamConsumer>::execute::{{closure}}::h7fa57d39bf140fb5

                                at /risingwave/src/stream/src/executor/dispatch.rs:392:9

   90:     0x5608c35536ea - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h7523a3d4ba1c4956

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   91:     0x5608c35536ea - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h0a96f0bb20d2c1ff

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   92:     0x5608c3304975 - <&mut S as futures_core::stream::Stream>::poll_next::hcc4e27b46f7caca2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:104:9

   93:     0x5608c3304975 - <tokio_stream::stream_ext::next::Next<St> as core::future::future::Future>::poll::h62490566a043917d

                                at /root/.cargo/git/checkouts/tokio-968c02b7a1a41bea-shallow/0dd1055/tokio-stream/src/stream_ext/next.rs:42:29

   94:     0x5608c3304975 - <tokio_stream::stream_ext::try_next::TryNext<St> as core::future::future::Future>::poll::ha1a44fb309a0d1fe

                                at /root/.cargo/git/checkouts/tokio-968c02b7a1a41bea-shallow/0dd1055/tokio-stream/src/stream_ext/try_next.rs:43:9

   95:     0x5608c3304975 - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h9d66efbe289da363

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9

   96:     0x5608c1d4cad4 - <await_tree::future::Instrumented<F,_> as core::future::future::Future>::poll::ha69c1dc43acecc2d

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/await-tree-0.2.1/src/future.rs:119:15

   97:     0x5608c1f4f139 - risingwave_stream::executor::actor::Actor<C>::run_consumer::{{closure}}::h60d08113ac7425e5

                                at /risingwave/src/stream/src/executor/actor.rs:250:18

   98:     0x5608c1f4f139 - <tokio::future::maybe_done::MaybeDone<Fut> as core::future::future::Future>::poll::hf47f6290805b8101

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/future/maybe_done.rs:62:56

   99:     0x5608c1f4f139 - risingwave_stream::executor::actor::Actor<C>::run::{{closure}}::{{closure}}::{{closure}}::hbfb6564d8059c5da

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/macros/join.rs:126:24

  100:     0x5608c1f4f139 - <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll::hf703258204d640e6

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/future/poll_fn.rs:58:9

  101:     0x5608c1f4f139 - risingwave_stream::executor::actor::Actor<C>::run::{{closure}}::{{closure}}::h875dc895f2c4af26

                                at /risingwave/src/stream/src/executor/actor.rs:194:13

  102:     0x5608c1e212b2 - <core::pin::Pin<P> as core::future::future::Future>::poll::h7aeb81031823b78f

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/future/future.rs:123:9

  103:     0x5608c1e212b2 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h3c1a0e8c4f3f88ad

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

  104:     0x5608c1e212b2 - tokio::task::task_local::LocalKey<T>::scope_inner::hd68834bd96b5ad9e

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

  105:     0x5608c1e212b2 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::h909cab4201bf1e3c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

  106:     0x5608c1e212b2 - risingwave_expr::expr_context::expr_context_scope::{{closure}}::he3d31ea862a3b697

                                at /risingwave/src/expr/core/src/expr_context.rs:43:65

  107:     0x5608c1e212b2 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::hc0412e3274a88dd6

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

  108:     0x5608c1e212b2 - tokio::task::task_local::LocalKey<T>::scope_inner::hdd0e083a00752559

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

  109:     0x5608c1e212b2 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::h464b9eb2df3e6cdd

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

  110:     0x5608c1e212b2 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h265cf0a74bb33c3f

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

  111:     0x5608c1e212b2 - tokio::task::task_local::LocalKey<T>::scope_inner::he4abe351b24d7699

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

  112:     0x5608c1e212b2 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::h2616f198c3b4e730

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

  113:     0x5608c378b8d4 - risingwave_stream::executor::actor::Actor<C>::run::{{closure}}::h524800449566125c

                                at /risingwave/src/stream/src/executor/actor.rs:208:13

  114:     0x5608c378b8d4 - <F as futures_core::future::TryFuture>::try_poll::hebab7356df8a24a9

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/future.rs:82:9

  115:     0x5608c378b8d4 - <futures_util::future::try_future::try_flatten::TryFlatten<Fut,<Fut as futures_core::future::TryFuture>::Ok> as core::future::future::Future>::poll::h4360a7e74a778a4b

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/try_future/try_flatten.rs:57:41

  116:     0x5608c26ac314 - <futures_util::future::try_future::TryFlatten<Fut1,Fut2> as core::future::future::Future>::poll::hbb64cd9a7685b668

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/lib.rs:91:13

  117:     0x5608c26ac314 - <futures_util::future::try_future::AndThen<Fut1,Fut2,F> as core::future::future::Future>::poll::h816cd4216e06eedf

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/lib.rs:91:13

  118:     0x5608c26ac314 - <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll::h4ec7baa2983f9f71

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/future/map.rs:55:37

  119:     0x5608c1e20f48 - <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll::hbef75e5c29e45b3b

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/lib.rs:91:13

  120:     0x5608c1e20f48 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h4bad4d189cdecc46

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

  121:     0x5608c1e20f48 - tokio::task::task_local::LocalKey<T>::scope_inner::ha18fbcbfb68f650f

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

  122:     0x5608c1e20f48 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::h0d388e5fde26a0ee

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

  123:     0x5608c3582685 - await_tree::root::TreeRoot::instrument::{{closure}}::h8f63df219ddb2d8c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/await-tree-0.2.1/src/root.rs:43:34

  124:     0x5608c3582685 - <futures_util::future::either::Either<A,B> as core::future::future::Future>::poll::ha41a032bf1b60af4

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/either.rs:109:32

  125:     0x5608c369be34 - core::ops::function::FnOnce::call_once::hbc22620692086941

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/ops/function.rs:250:5

  126:     0x5608c369be34 - tokio_metrics::task::instrument_poll::hc548122f29bac09f

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-metrics-0.3.1/src/task.rs:2530:15

  127:     0x5608c369be34 - <tokio_metrics::task::Instrumented<T> as core::future::future::Future>::poll::h07183b8cf42007dd

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-metrics-0.3.1/src/task.rs:2430:9

  128:     0x5608c1e21fcb - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h69f8dc244d9f4297

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

  129:     0x5608c1e21fcb - tokio::task::task_local::LocalKey<T>::scope_inner::hb645d06844924eae

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

  130:     0x5608c1e21fcb - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::hedf1595e14b24c2c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

  131:     0x5608c32ea7f6 - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h2965fd94ac60753c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9

  132:     0x5608c29091e3 - tokio::runtime::task::core::Core<T,S>::poll::{{closure}}::h544f1aaa5584f0dc

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:328:17

  133:     0x5608c29091e3 - tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut::hde542aa861dfb26c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9

  134:     0x5608c29091e3 - tokio::runtime::task::core::Core<T,S>::poll::h232b909e6ec85566

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:317:30

  135:     0x5608c2f4e535 - tokio::runtime::task::harness::poll_future::{{closure}}::h6082891c733c0082

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:485:19

  136:     0x5608c2f4e535 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h236bb6e286f5dfb3

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/panic/unwind_safe.rs:272:9

  137:     0x5608c2f4e535 - std::panicking::try::do_call::he0fdc48f83285c65

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:557:40

  138:     0x5608c2f4e535 - std::panicking::try::h9fa73b71cb8136ed

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:521:19

  139:     0x5608c2f4e535 - std::panic::catch_unwind::hd356c9f2c8fb75d5

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panic.rs:350:14

  140:     0x5608c2f4e535 - tokio::runtime::task::harness::poll_future::hcd46491544328d12

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:473:18

  141:     0x5608c2f4e535 - tokio::runtime::task::harness::Harness<T,S>::poll_inner::ha9a7d13061830067

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:208:27

  142:     0x5608c2f4e535 - tokio::runtime::task::harness::Harness<T,S>::poll::h64b8a4fb6ccc7a3d

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:153:15

  143:     0x5608c420f55a - tokio::runtime::task::raw::RawTask::poll::h341e16a2735e8961

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:201:18

  144:     0x5608c420f55a - tokio::runtime::task::LocalNotified<S>::run::hd5d5394e1efe87b5

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/mod.rs:427:9

  145:     0x5608c420f55a - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}::hec778c1ddf6933fe

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:585:18

  146:     0x5608c420f55a - tokio::runtime::coop::with_budget::h5efbb758872ecff5

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:107:5

  147:     0x5608c420f55a - tokio::runtime::coop::budget::h9b7d6fc5f3a388c5

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:73:5

  148:     0x5608c420f55a - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::h15848ef1bb81e812

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:584:9

  149:     0x5608c420ebe9 - tokio::runtime::scheduler::multi_thread::worker::Context::run::hb4f329fc2c7fb381

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:535:24

  150:     0x5608c41d4136 - tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}::h4dfb7b0d040c2ef9

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:500:21

  151:     0x5608c41d4136 - tokio::runtime::context::scoped::Scoped<T>::set::h3214d5cea78f05d6

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/scoped.rs:40:9

  152:     0x5608c41f48c8 - tokio::runtime::context::set_scheduler::{{closure}}::h984b860ee83825da

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context.rs:180:26

  153:     0x5608c41f48c8 - std::thread::local::LocalKey<T>::try_with::h871a6c3709bd0f9b

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/thread/local.rs:283:12

  154:     0x5608c41f48c8 - std::thread::local::LocalKey<T>::with::h9be64a7992e4a056

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/thread/local.rs:260:9

  155:     0x5608c41f48c8 - tokio::runtime::context::set_scheduler::hd68488b39fe372ac

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context.rs:180:17

  156:     0x5608c41f48c8 - tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::hc80eca66cb5c3c97

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:495:9

  157:     0x5608c41f48c8 - tokio::runtime::context::runtime::enter_runtime::h412d390839600b64

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/runtime.rs:65:16

  158:     0x5608c420e3cc - tokio::runtime::scheduler::multi_thread::worker::run::h30dbdeab05f7ef92

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:487:5

  159:     0x5608c420682f - tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}::hf23d7e1a55f8c143

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:455:45

  160:     0x5608c420682f - <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll::h63af7601cb24f6cf

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/task.rs:42:21

  161:     0x5608c420682f - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::ha023992f18098b1c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9

  162:     0x5608c41df346 - tokio::runtime::task::core::Core<T,S>::poll::{{closure}}::h2e9422173ba4ea74

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:328:17

  163:     0x5608c41df346 - tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut::h236bfe90733804c1

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9

  164:     0x5608c41df346 - tokio::runtime::task::core::Core<T,S>::poll::h6067acc8861db220

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:317:30

  165:     0x5608c41e300f - tokio::runtime::task::harness::poll_future::{{closure}}::hacdbafe644ffda0c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:485:19

  166:     0x5608c41e300f - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h13c0f21c5d32fad9

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/panic/unwind_safe.rs:272:9

  167:     0x5608c41e300f - std::panicking::try::do_call::h53cf9b1c334382ab

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:557:40

  168:     0x5608c41e300f - std::panicking::try::h6550e55105e111e2

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:521:19

  169:     0x5608c41e300f - std::panic::catch_unwind::h6e5f810b2c90df11

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panic.rs:350:14

  170:     0x5608c41e300f - tokio::runtime::task::harness::poll_future::h7e2ba866552ffd5e

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:473:18

  171:     0x5608c41e300f - tokio::runtime::task::harness::Harness<T,S>::poll_inner::h2cbc379ea659fada

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:208:27

  172:     0x5608c41e300f - tokio::runtime::task::harness::Harness<T,S>::poll::hb84d1e085aa0cabb

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:153:15

  173:     0x5608c41db4d2 - tokio::runtime::task::raw::RawTask::poll::h341e16a2735e8961

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:201:18

  174:     0x5608c41db4d2 - tokio::runtime::task::UnownedTask<S>::run::hd54d0259119fc5fa

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/mod.rs:464:9

  175:     0x5608c41db4d2 - tokio::runtime::blocking::pool::Task::run::hf34a8386f49075be

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:159:9

  176:     0x5608c41db4d2 - tokio::runtime::blocking::pool::Inner::run::h6a61e01289fe7cd5

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:513:17

  177:     0x5608c41dc7a0 - tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}::h89af5dbfab9410aa

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:471:13

  178:     0x5608c41dc7a0 - std::sys::backtrace::__rust_begin_short_backtrace::hd2047eff80f04515

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/sys/backtrace.rs:152:18

  179:     0x5608c41fb1ea - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hd81a94123c1d9c66

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/thread/mod.rs:542:17

  180:     0x5608c41fb1ea - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h7c2c6ec123bb57e3

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/panic/unwind_safe.rs:272:9

  181:     0x5608c41fb1ea - std::panicking::try::do_call::hd7847c03749886f4

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:557:40

  182:     0x5608c41fb1ea - std::panicking::try::h88fdcd0c4857923e

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:521:19

  183:     0x5608c41fb1ea - std::panic::catch_unwind::h0bb14717d9a623cf

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panic.rs:350:14

  184:     0x5608c41fb1ea - std::thread::Builder::spawn_unchecked_::{{closure}}::h61baa92b16dd20ea

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/thread/mod.rs:541:30

  185:     0x5608c41fb1ea - core::ops::function::FnOnce::call_once{{vtable.shim}}::he59ec4dfc46144f6

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/ops/function.rs:250:5

  186:     0x5608c4153f1b - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h62af1c252974e0db

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/alloc/src/boxed.rs:2070:9

  187:     0x5608c4153f1b - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h6e711421bc35ccc9

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/alloc/src/boxed.rs:2070:9

  188:     0x5608c4153f1b - std::sys::pal::unix::thread::Thread::new::thread_start::h70449abe2e248417

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/sys/pal/unix/thread.rs:108:17

  189:     0x7b40f3cb3a94 - start_thread

                                at ./nptl/pthread_create.c:447:8

  190:     0x7b40f3d40c3c - __GI___clone3

                                at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:78

  191:                0x0 - <unknown>

 *** await tree context of current task ***

 thread 'rw-streaming' panicked at src/stream/src/executor/temporal_join.rs:97:13:

 key OwnedRow([Some(Serial(Serial(490115337665433613))), Some(Int64(147511))]) should be in the cache

 stack backtrace:

    0:     0x5608c414c1d9 - std::backtrace_rs::backtrace::libunwind::trace::h81f95f911fafa2e0

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/../../backtrace/src/backtrace/libunwind.rs:116:5

    1:     0x5608c414c1d9 - std::backtrace_rs::backtrace::trace_unsynchronized::h2e88326ce80cfb26

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5

    2:     0x5608c414c1d9 - std::sys::backtrace::_print_fmt::h3318b90cf1bd0c10

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/sys/backtrace.rs:65:5

    3:     0x5608c414c1d9 - <std::sys::backtrace::BacktraceLock::print::DisplayBacktrace as core::fmt::Display>::fmt::hfb260f93257b2f2e

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/sys/backtrace.rs:40:26

    4:     0x5608b582ee5b - core::fmt::rt::Argument::fmt::h78c29ca6c560b840

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/fmt/rt.rs:173:76

    5:     0x5608b582ee5b - core::fmt::write::h70fe4701d8d7a171

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/fmt/mod.rs:1182:21

    6:     0x5608c41470bf - std::io::Write::write_fmt::h14783b7a2197dfce

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/io/mod.rs:1827:15

    7:     0x5608c414d301 - std::sys::backtrace::BacktraceLock::print::hc4ed4e400debb2bf

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/sys/backtrace.rs:43:9

    8:     0x5608c414d301 - std::panicking::default_hook::{{closure}}::hfff4b120f05a97bf

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:269:22

    9:     0x5608c414cfe7 - std::panicking::default_hook::h52349d4986dd35a0

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:296:9

   10:     0x5608c0f8311b - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::he9972cd62e1b6c03

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/alloc/src/boxed.rs:2084:9

   11:     0x5608c0f8311b - risingwave_rt::panic_hook::set_panic_hook::{{closure}}::hb4d8d816619ba0af

                                at /risingwave/src/utils/runtime/src/panic_hook.rs:25:9

   12:     0x5608c0f8311b - std::panicking::update_hook::{{closure}}::h1b85f23f2996e971

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:233:47

   13:     0x5608c414db6f - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::h3da72443baa8e06b

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/alloc/src/boxed.rs:2084:9

   14:     0x5608c414db6f - std::panicking::rust_panic_with_hook::hb85b5d3fee996073

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:808:13

   15:     0x5608c414d857 - std::panicking::begin_panic_handler::{{closure}}::hea92f85bb1e00425

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:674:13

   16:     0x5608c414c699 - std::sys::backtrace::__rust_end_short_backtrace::h4835e8f15136c715

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/sys/backtrace.rs:168:18

   17:     0x5608c414d4d3 - rust_begin_unwind

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:665:5

   18:     0x5608b582b3e2 - core::panicking::panic_fmt::hb0937d4ccfd82515

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/panicking.rs:74:14

   19:     0x5608c231d5c8 - risingwave_stream::executor::temporal_join::JoinEntry::remove::hf82f799c404dcf9a

                                at /risingwave/src/stream/src/executor/temporal_join.rs:97:13

   20:     0x5608c231d5c8 - risingwave_stream::executor::temporal_join::TemporalSide<K,S>::update::h2b7e834bea031c2a

                                at /risingwave/src/stream/src/executor/temporal_join.rs:194:58

   21:     0x5608c217327c - risingwave_stream::executor::temporal_join::TemporalJoinExecutor<K,S,_,_>::into_stream::{{closure}}::h981916f4fc949d9a

                                at /risingwave/src/stream/src/executor/temporal_join.rs:833:21

   22:     0x5608c217327c - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h9b75b85fd76a356f

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   23:     0x5608c1d6132a - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h89379077e6150140

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   24:     0x5608c1d6132a - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h38396531d651abe7

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   25:     0x5608c1d6132a - futures_util::stream::stream::StreamExt::poll_next_unpin::h154f4b34eab6b1fc

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9

   26:     0x5608c1d6132a - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h55b7892ee7493236

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9

   27:     0x5608c1d6132a - <await_tree::future::Instrumented<F,_> as core::future::future::Future>::poll::he063ee30d8e33e62

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/await-tree-0.2.1/src/future.rs:119:15

   28:     0x5608c34c2cef - risingwave_stream::executor::wrapper::trace::instrument_await_tree::{{closure}}::h65df7c64fe49803e

                                at /risingwave/src/stream/src/executor/wrapper/trace.rs:121:10

   29:     0x5608c34c2cef - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h4012cf30c241de43

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   30:     0x5608c34c2cef - risingwave_stream::executor::wrapper::schema_check::schema_check::{{closure}}::hcdaef53dfdb1c3b6

                                at /risingwave/src/stream/src/executor/wrapper/schema_check.rs:24:1

   31:     0x5608c34c2cef - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h8378a1d556b63d1a

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   32:     0x5608c34c2cef - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::hbced3fa0d3159b50

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   33:     0x5608c34c2cef - futures_util::stream::stream::StreamExt::poll_next_unpin::h2b17a56f54acd6a8

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9

   34:     0x5608c34bda8c - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h7f0ffc044dac8996

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9

   35:     0x5608c34bda8c - risingwave_stream::executor::wrapper::epoch_check::epoch_check::{{closure}}::h98262a88b11c76e7

                                at /risingwave/src/stream/src/executor/wrapper/epoch_check.rs:31:44

   36:     0x5608c34bda8c - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::hfcf54f81b20696f2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   37:     0x5608c34bda8c - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::hf0f90fee65c4d60a

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   38:     0x5608c34bda8c - <S as futures_core::stream::TryStream>::try_poll_next::h9ffa6315b1cc546a

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:196:9

   39:     0x5608c34bda8c - futures_util::stream::try_stream::TryStreamExt::try_poll_next_unpin::he56f26f8a8be87ab

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/try_stream/mod.rs:1131:9

   40:     0x5608c1e21cc8 - <futures_util::stream::try_stream::try_next::TryNext<St> as core::future::future::Future>::poll::h74746373b1039b83

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/try_stream/try_next.rs:32:9

 Actor 6776: `` [13.462s]

   Epoch 7658170125975552 [576.999ms]

     Filter 1A7800000010 [290.000ms]

       TemporalJoin 1A780000000F [290.000ms]  <== current

   41:     0x5608c1e21cc8 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h0e1e4bd427b12e86

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

   42:     0x5608c1e21cc8 - tokio::task::task_local::LocalKey<T>::scope_inner::h7edb0552146c04ac

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

   43:     0x5608c1e21cc8 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::hd7d34eb12d8caab2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

   44:     0x5608c34c381c - risingwave_common::util::epoch::task_local::scope::{{closure}}::hab192e60dda84fbb

                                at /risingwave/src/common/src/util/epoch.rs:249:47

   45:     0x5608c34c381c - risingwave_stream::executor::wrapper::epoch_provide::epoch_provide::{{closure}}::h7ab6553c8c5b1b7b

                                at /risingwave/src/stream/src/executor/wrapper/epoch_provide.rs:31:59

   46:     0x5608c34c381c - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h5a83fafe411e9554

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   47:     0x5608c34c381c - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h10d754f96fe381ec

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   48:     0x5608c34c381c - futures_util::stream::stream::StreamExt::poll_next_unpin::h4401b57aaca2d4f2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9

   49:     0x5608c32ec6cc - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h1c1f040471113424

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9

   50:     0x5608c32ec6cc - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h369eb8c15e0a31ea

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9

   51:     0x5608c1e2c28a - risingwave_stream::executor::wrapper::trace::trace::{{closure}}::h5a79a0a254b17b9e

                                at /risingwave/src/stream/src/executor/wrapper/trace.rs:58:69

   52:     0x5608c1e2c28a - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h4c5f95930c499f9c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   53:     0x5608c35542d1 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h19c2117822980a29

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   54:     0x5608c299d58f - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h89379077e6150140

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   55:     0x5608c299d58f - risingwave_stream::executor::filter::FilterExecutor::execute_inner::{{closure}}::he75eca12ddfe9559

                                at /risingwave/src/stream/src/executor/filter.rs:134:5

   56:     0x5608c299d58f - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h5a39b90262b30141

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   57:     0x5608c1d6132a - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h89379077e6150140

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   58:     0x5608c1d6132a - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h38396531d651abe7

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   59:     0x5608c1d6132a - futures_util::stream::stream::StreamExt::poll_next_unpin::h154f4b34eab6b1fc

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9

   60:     0x5608c1d6132a - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h55b7892ee7493236

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9

   61:     0x5608c1d6132a - <await_tree::future::Instrumented<F,_> as core::future::future::Future>::poll::he063ee30d8e33e62

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/await-tree-0.2.1/src/future.rs:119:15

   62:     0x5608c34c2cef - risingwave_stream::executor::wrapper::trace::instrument_await_tree::{{closure}}::h65df7c64fe49803e

                                at /risingwave/src/stream/src/executor/wrapper/trace.rs:121:10

   63:     0x5608c34c2cef - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h4012cf30c241de43

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   64:     0x5608c34c2cef - risingwave_stream::executor::wrapper::schema_check::schema_check::{{closure}}::hcdaef53dfdb1c3b6

                                at /risingwave/src/stream/src/executor/wrapper/schema_check.rs:24:1

   65:     0x5608c34c2cef - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h8378a1d556b63d1a

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   66:     0x5608c34c2cef - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::hbced3fa0d3159b50

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   67:     0x5608c34c2cef - futures_util::stream::stream::StreamExt::poll_next_unpin::h2b17a56f54acd6a8

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9

   68:     0x5608c34bda8c - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h7f0ffc044dac8996

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9

   69:     0x5608c34bda8c - risingwave_stream::executor::wrapper::epoch_check::epoch_check::{{closure}}::h98262a88b11c76e7

                                at /risingwave/src/stream/src/executor/wrapper/epoch_check.rs:31:44

   70:     0x5608c34bda8c - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::hfcf54f81b20696f2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   71:     0x5608c34bda8c - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::hf0f90fee65c4d60a

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   72:     0x5608c34bda8c - <S as futures_core::stream::TryStream>::try_poll_next::h9ffa6315b1cc546a

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:196:9

   73:     0x5608c34bda8c - futures_util::stream::try_stream::TryStreamExt::try_poll_next_unpin::he56f26f8a8be87ab

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/try_stream/mod.rs:1131:9

   74:     0x5608c1e21cc8 - <futures_util::stream::try_stream::try_next::TryNext<St> as core::future::future::Future>::poll::h74746373b1039b83

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/try_stream/try_next.rs:32:9

   75:     0x5608c1e21cc8 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h0e1e4bd427b12e86

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

   76:     0x5608c1e21cc8 - tokio::task::task_local::LocalKey<T>::scope_inner::h7edb0552146c04ac

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

   77:     0x5608c1e21cc8 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::hd7d34eb12d8caab2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

   78:     0x5608c34c381c - risingwave_common::util::epoch::task_local::scope::{{closure}}::hab192e60dda84fbb

                                at /risingwave/src/common/src/util/epoch.rs:249:47

   79:     0x5608c34c381c - risingwave_stream::executor::wrapper::epoch_provide::epoch_provide::{{closure}}::h7ab6553c8c5b1b7b

                                at /risingwave/src/stream/src/executor/wrapper/epoch_provide.rs:31:59

   80:     0x5608c34c381c - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h5a83fafe411e9554

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   81:     0x5608c34c381c - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h10d754f96fe381ec

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   82:     0x5608c34c381c - futures_util::stream::stream::StreamExt::poll_next_unpin::h4401b57aaca2d4f2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9

   83:     0x5608c32ec6cc - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h1c1f040471113424

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9

   84:     0x5608c32ec6cc - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h369eb8c15e0a31ea

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9

   85:     0x5608c1e2c28a - risingwave_stream::executor::wrapper::trace::trace::{{closure}}::h5a79a0a254b17b9e

                                at /risingwave/src/stream/src/executor/wrapper/trace.rs:58:69

   86:     0x5608c1e2c28a - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h4c5f95930c499f9c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   87:     0x5608c35542d1 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h19c2117822980a29

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   88:     0x5608c35536ea - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h89379077e6150140

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   89:     0x5608c35536ea - <risingwave_stream::executor::dispatch::DispatchExecutor as risingwave_stream::executor::StreamConsumer>::execute::{{closure}}::h7fa57d39bf140fb5

                                at /risingwave/src/stream/src/executor/dispatch.rs:392:9

   90:     0x5608c35536ea - <futures_async_stream::try_stream::GenTryStream<G> as futures_core::stream::Stream>::poll_next::h7523a3d4ba1c4956

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-async-stream-0.2.11/src/lib.rs:492:33

   91:     0x5608c35536ea - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h0a96f0bb20d2c1ff

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9

   92:     0x5608c3304975 - <&mut S as futures_core::stream::Stream>::poll_next::hcc4e27b46f7caca2

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:104:9

   93:     0x5608c3304975 - <tokio_stream::stream_ext::next::Next<St> as core::future::future::Future>::poll::h62490566a043917d

                                at /root/.cargo/git/checkouts/tokio-968c02b7a1a41bea-shallow/0dd1055/tokio-stream/src/stream_ext/next.rs:42:29

   94:     0x5608c3304975 - <tokio_stream::stream_ext::try_next::TryNext<St> as core::future::future::Future>::poll::ha1a44fb309a0d1fe

                                at /root/.cargo/git/checkouts/tokio-968c02b7a1a41bea-shallow/0dd1055/tokio-stream/src/stream_ext/try_next.rs:43:9

   95:     0x5608c3304975 - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h9d66efbe289da363

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9

   96:     0x5608c1d4cad4 - <await_tree::future::Instrumented<F,_> as core::future::future::Future>::poll::ha69c1dc43acecc2d

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/await-tree-0.2.1/src/future.rs:119:15

   97:     0x5608c1f4f139 - risingwave_stream::executor::actor::Actor<C>::run_consumer::{{closure}}::h60d08113ac7425e5

                                at /risingwave/src/stream/src/executor/actor.rs:250:18

   98:     0x5608c1f4f139 - <tokio::future::maybe_done::MaybeDone<Fut> as core::future::future::Future>::poll::hf47f6290805b8101

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/future/maybe_done.rs:62:56

   99:     0x5608c1f4f139 - risingwave_stream::executor::actor::Actor<C>::run::{{closure}}::{{closure}}::{{closure}}::hbfb6564d8059c5da

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/macros/join.rs:126:24

  100:     0x5608c1f4f139 - <tokio::future::poll_fn::PollFn<F> as core::future::future::Future>::poll::hf703258204d640e6

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/future/poll_fn.rs:58:9

  101:     0x5608c1f4f139 - risingwave_stream::executor::actor::Actor<C>::run::{{closure}}::{{closure}}::h875dc895f2c4af26

                                at /risingwave/src/stream/src/executor/actor.rs:194:13

  102:     0x5608c1e212b2 - <core::pin::Pin<P> as core::future::future::Future>::poll::h7aeb81031823b78f

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/future/future.rs:123:9

  103:     0x5608c1e212b2 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h3c1a0e8c4f3f88ad

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

  104:     0x5608c1e212b2 - tokio::task::task_local::LocalKey<T>::scope_inner::hd68834bd96b5ad9e

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

  105:     0x5608c1e212b2 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::h909cab4201bf1e3c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

  106:     0x5608c1e212b2 - risingwave_expr::expr_context::expr_context_scope::{{closure}}::he3d31ea862a3b697

                                at /risingwave/src/expr/core/src/expr_context.rs:43:65

  107:     0x5608c1e212b2 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::hc0412e3274a88dd6

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

  108:     0x5608c1e212b2 - tokio::task::task_local::LocalKey<T>::scope_inner::hdd0e083a00752559

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

  109:     0x5608c1e212b2 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::h464b9eb2df3e6cdd

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

  110:     0x5608c1e212b2 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h265cf0a74bb33c3f

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

  111:     0x5608c1e212b2 - tokio::task::task_local::LocalKey<T>::scope_inner::he4abe351b24d7699

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

  112:     0x5608c1e212b2 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::h2616f198c3b4e730

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

  113:     0x5608c378b8d4 - risingwave_stream::executor::actor::Actor<C>::run::{{closure}}::h524800449566125c

                                at /risingwave/src/stream/src/executor/actor.rs:208:13

  114:     0x5608c378b8d4 - <F as futures_core::future::TryFuture>::try_poll::hebab7356df8a24a9

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/future.rs:82:9

  115:     0x5608c378b8d4 - <futures_util::future::try_future::try_flatten::TryFlatten<Fut,<Fut as futures_core::future::TryFuture>::Ok> as core::future::future::Future>::poll::h4360a7e74a778a4b

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/try_future/try_flatten.rs:57:41

  116:     0x5608c26ac314 - <futures_util::future::try_future::TryFlatten<Fut1,Fut2> as core::future::future::Future>::poll::hbb64cd9a7685b668

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/lib.rs:91:13

  117:     0x5608c26ac314 - <futures_util::future::try_future::AndThen<Fut1,Fut2,F> as core::future::future::Future>::poll::h816cd4216e06eedf

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/lib.rs:91:13

  118:     0x5608c26ac314 - <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll::h4ec7baa2983f9f71

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/future/map.rs:55:37

  119:     0x5608c1e20f48 - <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll::hbef75e5c29e45b3b

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/lib.rs:91:13

  120:     0x5608c1e20f48 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h4bad4d189cdecc46

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

  121:     0x5608c1e20f48 - tokio::task::task_local::LocalKey<T>::scope_inner::ha18fbcbfb68f650f

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

  122:     0x5608c1e20f48 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::h0d388e5fde26a0ee

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

  123:     0x5608c3582685 - await_tree::root::TreeRoot::instrument::{{closure}}::h8f63df219ddb2d8c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/await-tree-0.2.1/src/root.rs:43:34

  124:     0x5608c3582685 - <futures_util::future::either::Either<A,B> as core::future::future::Future>::poll::ha41a032bf1b60af4

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/either.rs:109:32

  125:     0x5608c369be34 - core::ops::function::FnOnce::call_once::hbc22620692086941

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/ops/function.rs:250:5

  126:     0x5608c369be34 - tokio_metrics::task::instrument_poll::hc548122f29bac09f

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-metrics-0.3.1/src/task.rs:2530:15

  127:     0x5608c369be34 - <tokio_metrics::task::Instrumented<T> as core::future::future::Future>::poll::h07183b8cf42007dd

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-metrics-0.3.1/src/task.rs:2430:9

  128:     0x5608c1e21fcb - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h69f8dc244d9f4297

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:391:31

  129:     0x5608c1e21fcb - tokio::task::task_local::LocalKey<T>::scope_inner::hb645d06844924eae

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:217:19

  130:     0x5608c1e21fcb - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::hedf1595e14b24c2c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/task/task_local.rs:387:19

  131:     0x5608c32ea7f6 - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h2965fd94ac60753c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9

  132:     0x5608c29091e3 - tokio::runtime::task::core::Core<T,S>::poll::{{closure}}::h544f1aaa5584f0dc

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:328:17

  133:     0x5608c29091e3 - tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut::hde542aa861dfb26c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9

  134:     0x5608c29091e3 - tokio::runtime::task::core::Core<T,S>::poll::h232b909e6ec85566

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:317:30

  135:     0x5608c2f4e535 - tokio::runtime::task::harness::poll_future::{{closure}}::h6082891c733c0082

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:485:19

  136:     0x5608c2f4e535 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h236bb6e286f5dfb3

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/panic/unwind_safe.rs:272:9

  137:     0x5608c2f4e535 - std::panicking::try::do_call::he0fdc48f83285c65

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:557:40

  138:     0x5608c2f4e535 - std::panicking::try::h9fa73b71cb8136ed

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:521:19

  139:     0x5608c2f4e535 - std::panic::catch_unwind::hd356c9f2c8fb75d5

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panic.rs:350:14

  140:     0x5608c2f4e535 - tokio::runtime::task::harness::poll_future::hcd46491544328d12

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:473:18

  141:     0x5608c2f4e535 - tokio::runtime::task::harness::Harness<T,S>::poll_inner::ha9a7d13061830067

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:208:27

  142:     0x5608c2f4e535 - tokio::runtime::task::harness::Harness<T,S>::poll::h64b8a4fb6ccc7a3d

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:153:15

  143:     0x5608c420f55a - tokio::runtime::task::raw::RawTask::poll::h341e16a2735e8961

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:201:18

  144:     0x5608c420f55a - tokio::runtime::task::LocalNotified<S>::run::hd5d5394e1efe87b5

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/mod.rs:427:9

  145:     0x5608c420f55a - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}::hec778c1ddf6933fe

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:585:18

  146:     0x5608c420f55a - tokio::runtime::coop::with_budget::h5efbb758872ecff5

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:107:5

  147:     0x5608c420f55a - tokio::runtime::coop::budget::h9b7d6fc5f3a388c5

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:73:5

  148:     0x5608c420f55a - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::h15848ef1bb81e812

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:584:9

  149:     0x5608c420ebe9 - tokio::runtime::scheduler::multi_thread::worker::Context::run::hb4f329fc2c7fb381

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:535:24

  150:     0x5608c41d4136 - tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}::h4dfb7b0d040c2ef9

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:500:21

  151:     0x5608c41d4136 - tokio::runtime::context::scoped::Scoped<T>::set::h3214d5cea78f05d6

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/scoped.rs:40:9

  152:     0x5608c41f48c8 - tokio::runtime::context::set_scheduler::{{closure}}::h984b860ee83825da

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context.rs:180:26

  153:     0x5608c41f48c8 - std::thread::local::LocalKey<T>::try_with::h871a6c3709bd0f9b

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/thread/local.rs:283:12

  154:     0x5608c41f48c8 - std::thread::local::LocalKey<T>::with::h9be64a7992e4a056

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/thread/local.rs:260:9

  155:     0x5608c41f48c8 - tokio::runtime::context::set_scheduler::hd68488b39fe372ac

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context.rs:180:17

  156:     0x5608c41f48c8 - tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::hc80eca66cb5c3c97

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:495:9

  157:     0x5608c41f48c8 - tokio::runtime::context::runtime::enter_runtime::h412d390839600b64

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/runtime.rs:65:16

  158:     0x5608c420e3cc - tokio::runtime::scheduler::multi_thread::worker::run::h30dbdeab05f7ef92

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:487:5

  159:     0x5608c420682f - tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}::hf23d7e1a55f8c143

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:455:45

  160:     0x5608c420682f - <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll::h63af7601cb24f6cf

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/task.rs:42:21

  161:     0x5608c420682f - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::ha023992f18098b1c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9

  162:     0x5608c41df346 - tokio::runtime::task::core::Core<T,S>::poll::{{closure}}::h2e9422173ba4ea74

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:328:17

  163:     0x5608c41df346 - tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut::h236bfe90733804c1

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9

  164:     0x5608c41df346 - tokio::runtime::task::core::Core<T,S>::poll::h6067acc8861db220

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:317:30

  165:     0x5608c41e300f - tokio::runtime::task::harness::poll_future::{{closure}}::hacdbafe644ffda0c

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:485:19

  166:     0x5608c41e300f - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h13c0f21c5d32fad9

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/panic/unwind_safe.rs:272:9

  167:     0x5608c41e300f - std::panicking::try::do_call::h53cf9b1c334382ab

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:557:40

  168:     0x5608c41e300f - std::panicking::try::h6550e55105e111e2

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:521:19

  169:     0x5608c41e300f - std::panic::catch_unwind::h6e5f810b2c90df11

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panic.rs:350:14

  170:     0x5608c41e300f - tokio::runtime::task::harness::poll_future::h7e2ba866552ffd5e

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:473:18

  171:     0x5608c41e300f - tokio::runtime::task::harness::Harness<T,S>::poll_inner::h2cbc379ea659fada

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:208:27

  172:     0x5608c41e300f - tokio::runtime::task::harness::Harness<T,S>::poll::hb84d1e085aa0cabb

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:153:15

  173:     0x5608c41db4d2 - tokio::runtime::task::raw::RawTask::poll::h341e16a2735e8961

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:201:18

  174:     0x5608c41db4d2 - tokio::runtime::task::UnownedTask<S>::run::hd54d0259119fc5fa

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/mod.rs:464:9

  175:     0x5608c41db4d2 - tokio::runtime::blocking::pool::Task::run::hf34a8386f49075be

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:159:9

  176:     0x5608c41db4d2 - tokio::runtime::blocking::pool::Inner::run::h6a61e01289fe7cd5

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:513:17

  177:     0x5608c41dc7a0 - tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}::h89af5dbfab9410aa

                                at /root/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:471:13

  178:     0x5608c41dc7a0 - std::sys::backtrace::__rust_begin_short_backtrace::hd2047eff80f04515

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/sys/backtrace.rs:152:18

  179:     0x5608c41fb1ea - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::hd81a94123c1d9c66

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/thread/mod.rs:542:17

  180:     0x5608c41fb1ea - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h7c2c6ec123bb57e3

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/panic/unwind_safe.rs:272:9

  181:     0x5608c41fb1ea - std::panicking::try::do_call::hd7847c03749886f4

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:557:40

  182:     0x5608c41fb1ea - std::panicking::try::h88fdcd0c4857923e

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panicking.rs:521:19

  183:     0x5608c41fb1ea - std::panic::catch_unwind::h0bb14717d9a623cf

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/panic.rs:350:14

  184:     0x5608c41fb1ea - std::thread::Builder::spawn_unchecked_::{{closure}}::h61baa92b16dd20ea

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/thread/mod.rs:541:30

  185:     0x5608c41fb1ea - core::ops::function::FnOnce::call_once{{vtable.shim}}::he59ec4dfc46144f6

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/core/src/ops/function.rs:250:5

  186:     0x5608c4153f1b - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h62af1c252974e0db

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/alloc/src/boxed.rs:2070:9

  187:     0x5608c4153f1b - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h6e711421bc35ccc9

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/alloc/src/boxed.rs:2070:9

  188:     0x5608c4153f1b - std::sys::pal::unix::thread::Thread::new::thread_start::h70449abe2e248417

                                at /rustc/5affbb17153bc69a9d5d8d2faa4e399a014a211e/library/std/src/sys/pal/unix/thread.rs:108:17

  189:     0x7b40f3cb3a94 - start_thread

 *** await tree context of current task ***

 Actor 6775: `` [13.476s]

   Epoch 7658170125975552 [583.999ms]

     Filter 1A7700000010 [295.000ms]

       TemporalJoin 1A770000000F [295.000ms]  <== current

                                at ./nptl/pthread_create.c:447:8

  190:     0x7b40f3d40c3c - __GI___clone3

                                at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:78

  191:                0x0 - <unknown>

 thread 'rw-streaming' panicked at src/stream/src/executor/temporal_join.rs:97:13:

 key OwnedRow([Some(Serial(Serial(490115338974072833))), Some(Int64(962654))]) should be in the cache

To Reproduce

CREATE MATERIALIZED VIEW mv1 AS
SELECT
  id,
....

CREATE INDEX mv1_id ON mv1(id);

CREATE MATERIALIZED VIEW mv2 AS
  SELECT
  id,
  ...
LEFT JOIN mv1 FOR SYSTEM_TIME AS OF PROCTIME() ON mv1.id = mv2.id
  ...

After 5 min - restart node

Expected behavior

No response

How did you deploy RisingWave?

RisingWave Helm

The version of RisingWave

PostgreSQL 13.14.0-RisingWave-2.1.0 (d6e98acc5c1faf815855a313e2464d9a4876ff4f)

And RisingWave-2.0.8 too

Additional context

No response