Add adaptive_bar_ordering to BacktestVenueConfig (#2188)

examples/backtest/databento_test_request_bars.py

@@ -6,7 +6,7 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.16.4
+#       jupytext_version: 1.16.6
 #   kernelspec:
 #     display_name: Python 3 (ipykernel)
 #     language: python
@@ -225,19 +225,22 @@ def user_log(self, msg):
 
 # %%
 # BacktestEngineConfig
+tested_market_data = "bars"  # or "quotes" or "trades"
+
+historical_start_delay = 10 if tested_market_data == "bars" else 2
+historical_end_delay = 1 if tested_market_data == "bars" else 0
+
+backtest_start = "2024-07-01T23:55" if tested_market_data == "bars" else "2024-07-02T00:00"
+backtest_end = "2024-07-02T00:10" if tested_market_data == "bars" else "2024-07-02T00:02"
 
 strategies = [
     ImportableStrategyConfig(
         strategy_path=TestHistoricalAggStrategy.fully_qualified_name(),
         config_path=TestHistoricalAggConfig.fully_qualified_name(),
         config={
             "symbol_id": InstrumentId.from_str(f"{future_symbols[0]}.GLBX"),
-            # for bars
-            "historical_start_delay": 10,
-            "historical_end_delay": 1,
-            # for quotes
-            # "historical_start_delay": 2,
-            # "historical_end_delay": 0,
+            "historical_start_delay": historical_start_delay,
+            "historical_end_delay": historical_end_delay,
         },
     ),
 ]
@@ -248,7 +251,7 @@ def user_log(self, msg):
     log_level="WARN",
     log_level_file="WARN",
     log_directory=".",
-    log_file_format=None,  # 'json' or None
+    log_file_format=None,  # "json" or None
     log_file_name="databento_option_greeks",
     clear_log_file=True,
 )
@@ -315,12 +318,8 @@ def user_log(self, msg):
         data=data,
         venues=venues,
         chunk_size=None,  # use None when loading custom data
-        # for bars
-        start="2024-07-01T23:55",
-        end="2024-07-02T00:10",
-        # for quotes or trades
-        # start="2024-07-02T00:00",
-        # end="2024-07-02T00:02",
+        start=backtest_start,
+        end=backtest_end,
     ),
 ]
 

nautilus_trader/backtest/config.py

@@ -110,6 +110,11 @@ class BacktestVenueConfig(NautilusConfig, frozen=True):
         If all venue generated identifiers will be random UUID4's.
     use_reduce_only : bool, default True
         If the `reduce_only` execution instruction on orders will be honored.
+    adaptive_bar_ordering : bool, default False
+        If High or Low should be processed first depending on distance with Open when using bars with the order matching engine.
+        If False then the processing order is always Open, High, Low, Close.
+        If High is closer to Open than Low then the processing order is Open, High, Low, Close.
+        If Low is closer to Open than High then the processing order is Open, Low, High, Close.
 
     """
 
@@ -131,6 +136,7 @@ class BacktestVenueConfig(NautilusConfig, frozen=True):
     use_position_ids: bool = True
     use_random_ids: bool = False
     use_reduce_only: bool = True
+    adaptive_bar_ordering: bool = False
     # fill_model: FillModel | None = None  # TODO: Implement
     modules: list[ImportableActorConfig] | None = None
 

nautilus_trader/backtest/engine.pyx

@@ -402,6 +402,7 @@ cdef class BacktestEngine:
         use_position_ids: bool = True,
         use_random_ids: bool = False,
         use_reduce_only: bool = True,
+        adaptive_bar_ordering: bool = False,
     ) -> None:
         """
         Add a `SimulatedExchange` with the given parameters to the backtest engine.
@@ -454,6 +455,11 @@ cdef class BacktestEngine:
             If all venue generated identifiers will be random UUID4's.
         use_reduce_only : bool, default True
             If the `reduce_only` execution instruction on orders will be honored.
+        adaptive_bar_ordering : bool, default False
+            If High or Low should be processed first depending on distance with Open when using bars with the order matching engine.
+            If False then the processing order is always Open, High, Low, Close.
+            If High is closer to Open than Low then the processing order is Open, High, Low, Close.
+            If Low is closer to Open than High then the processing order is Open, Low, High, Close.
 
         Raises
         ------
@@ -507,6 +513,7 @@ cdef class BacktestEngine:
             use_position_ids=use_position_ids,
             use_random_ids=use_random_ids,
             use_reduce_only=use_reduce_only,
+            adaptive_bar_ordering=adaptive_bar_ordering,
         )
 
         self._venues[venue] = exchange

nautilus_trader/backtest/exchange.pxd

@@ -103,6 +103,8 @@ cdef class SimulatedExchange:
     """The simulation modules registered with the exchange.\n\n:returns: `list[SimulationModule]`"""
     cdef readonly dict instruments
     """The exchange instruments.\n\n:returns: `dict[InstrumentId, Instrument]`"""
+    cdef readonly bint adaptive_bar_ordering
+    """If High or Low should be processed first depending on distance with Open when using bars with the order matching engine.\n\n:returns: `bool`"""
 
     cdef dict _matching_engines
     cdef object _message_queue

nautilus_trader/backtest/exchange.pyx

@@ -124,6 +124,11 @@ cdef class SimulatedExchange:
         they have initially arrived. Setting this to False would be appropriate for real-time
         sandbox environments, where we don't want to introduce additional latency of waiting for
         the next data event before processing the trading command.
+    adaptive_bar_ordering : bool, default False
+        If High or Low should be processed first depending on distance with Open when using bars with the order matching engine.
+        If False then the processing order is always Open, High, Low, Close.
+        If High is closer to Open than Low then the processing order is Open, High, Low, Close.
+        If Low is closer to Open than High then the processing order is Open, Low, High, Close.
 
     Raises
     ------
@@ -170,6 +175,7 @@ cdef class SimulatedExchange:
         bint use_random_ids = False,
         bint use_reduce_only = True,
         bint use_message_queue = True,
+        bint adaptive_bar_ordering = False,
     ) -> None:
         Condition.not_empty(starting_balances, "starting_balances")
         Condition.list_type(starting_balances, Money, "starting_balances")
@@ -212,6 +218,7 @@ cdef class SimulatedExchange:
         self.fill_model = fill_model
         self.fee_model = fee_model
         self.latency_model = latency_model
+        self.adaptive_bar_ordering = adaptive_bar_ordering
 
         # Load modules
         self.modules = []
@@ -356,6 +363,7 @@ cdef class SimulatedExchange:
             use_position_ids=self.use_position_ids,
             use_random_ids=self.use_random_ids,
             use_reduce_only=self.use_reduce_only,
+            adaptive_bar_ordering=self.adaptive_bar_ordering,
         )
 
         self._matching_engines[instrument.id] = matching_engine

nautilus_trader/backtest/matching_engine.pxd

@@ -92,6 +92,7 @@ cdef class OrderMatchingEngine:
     cdef bint _use_position_ids
     cdef bint _use_random_ids
     cdef bint _use_reduce_only
+    cdef bint _adaptive_bar_ordering
     cdef dict _account_ids
     cdef dict _execution_bar_types
     cdef dict _execution_bar_deltas
@@ -152,7 +153,17 @@ cdef class OrderMatchingEngine:
     cpdef void process_auction_book(self, OrderBook book)
     cpdef void process_instrument_close(self, InstrumentClose close)
     cdef void _process_trade_ticks_from_bar(self, Bar bar)
+    cdef TradeTick _create_base_trade_tick(self, Bar bar, Quantity size)
+    cdef void _process_trade_bar_open(self, Bar bar, TradeTick tick)
+    cdef void _process_trade_bar_high(self, Bar bar, TradeTick tick)
+    cdef void _process_trade_bar_low(self, Bar bar, TradeTick tick)
+    cdef void _process_trade_bar_close(self, Bar bar, TradeTick tick)
     cdef void _process_quote_ticks_from_bar(self)
+    cdef QuoteTick _create_base_quote_tick(self, Quantity bid_size, Quantity ask_size)
+    cdef void _process_quote_bar_open(self, QuoteTick tick)
+    cdef void _process_quote_bar_high(self, QuoteTick tick)
+    cdef void _process_quote_bar_low(self, QuoteTick tick)
+    cdef void _process_quote_bar_close(self, QuoteTick tick)
 
 # -- TRADING COMMANDS -----------------------------------------------------------------------------
 

nautilus_trader/backtest/matching_engine.pyx

@@ -157,6 +157,11 @@ cdef class OrderMatchingEngine:
         If the `reduce_only` execution instruction on orders will be honored.
     auction_match_algo : Callable[[Ladder, Ladder], Tuple[List, List], optional
         The auction matching algorithm.
+    adaptive_bar_ordering : bool, default False
+        If High or Low should be processed first depending on distance with Open when using bars with the order matching engine.
+        If False then the processing order is always Open, High, Low, Close.
+        If High is closer to Open than Low then the processing order is Open, High, Low, Close.
+        If Low is closer to Open than High then the processing order is Open, Low, High, Close.
 
     """
 
@@ -180,6 +185,7 @@ cdef class OrderMatchingEngine:
         bint use_position_ids = True,
         bint use_random_ids = False,
         bint use_reduce_only = True,
+        bint adaptive_bar_ordering = False,
         # auction_match_algo = default_auction_match
     ) -> None:
         self._clock = clock
@@ -205,6 +211,7 @@ cdef class OrderMatchingEngine:
         self._use_position_ids = use_position_ids
         self._use_random_ids = use_random_ids
         self._use_reduce_only = use_reduce_only
+        self._adaptive_bar_ordering = adaptive_bar_ordering
         # self._auction_match_algo = auction_match_algo
         self._fill_model = fill_model
         self._fee_model = fee_model
@@ -635,8 +642,23 @@ cdef class OrderMatchingEngine:
         cdef double size_value = max(bar.volume.as_double() / 4.0, self.instrument.size_increment.as_double())
         cdef Quantity size = Quantity(size_value, bar._mem.volume.precision)
 
-        # Create reusable tick
-        cdef TradeTick tick = TradeTick(
+        # Create base tick template
+        cdef TradeTick tick = self._create_base_trade_tick(bar, size)
+
+        # Process each price point
+        cdef bint process_high_first = not self._adaptive_bar_ordering or abs(bar._mem.high.raw - bar._mem.open.raw) < abs(bar._mem.low.raw - bar._mem.open.raw)
+
+        self._process_trade_bar_open(bar, tick)
+        if process_high_first:
+            self._process_trade_bar_high(bar, tick)
+            self._process_trade_bar_low(bar, tick)
+        else:
+            self._process_trade_bar_low(bar, tick)
+            self._process_trade_bar_high(bar, tick)
+        self._process_trade_bar_close(bar, tick)
+
+    cdef TradeTick _create_base_trade_tick(self, Bar bar, Quantity size):
+        return TradeTick(
             bar.bar_type.instrument_id,
             bar.open,
             size,
@@ -646,51 +668,43 @@ cdef class OrderMatchingEngine:
             bar.ts_event,
         )
 
-        if is_logging_initialized():
-            self._log.debug(f"Processing trade from bar {tick!r}")
-
-        # Open
-        if not self._core.is_last_initialized or bar._mem.open.raw != self._core.last_raw:  # Direct memory comparison
+    cdef void _process_trade_bar_open(self, Bar bar, TradeTick tick):
+        if not self._core.is_last_initialized or bar._mem.open.raw != self._core.last_raw:
             if is_logging_initialized():
                 self._log.debug(f"Updating with open {bar.open}")
             self._book.update_trade_tick(tick)
             self.iterate(tick.ts_init)
             self._core.set_last_raw(bar._mem.open.raw)
 
-        cdef str trade_id_str  # Assigned below
-
-        # High
-        if bar._mem.high.raw > self._core.last_raw:  # Direct memory comparison
+    cdef void _process_trade_bar_high(self, Bar bar, TradeTick tick):
+        if bar._mem.high.raw > self._core.last_raw:
             if is_logging_initialized():
                 self._log.debug(f"Updating with high {bar.high}")
-            tick._mem.price = bar._mem.high  # Direct memory assignment
-            tick._mem.aggressor_side = AggressorSide.BUYER  # Direct memory assignment
-            trade_id_str = self._generate_trade_id_str()
-            tick._mem.trade_id = trade_id_new(pystr_to_cstr(trade_id_str))
+            tick._mem.price = bar._mem.high
+            tick._mem.aggressor_side = AggressorSide.BUYER
+            tick._mem.trade_id = trade_id_new(pystr_to_cstr(self._generate_trade_id_str()))
             self._book.update_trade_tick(tick)
             self.iterate(tick.ts_init)
             self._core.set_last_raw(bar._mem.high.raw)
 
-        # Low
-        if bar._mem.low.raw < self._core.last_raw:  # Direct memory comparison
+    cdef void _process_trade_bar_low(self, Bar bar, TradeTick tick):
+        if bar._mem.low.raw < self._core.last_raw:
             if is_logging_initialized():
                 self._log.debug(f"Updating with low {bar.low}")
-            tick._mem.price = bar._mem.low  # Direct memory assignment
+            tick._mem.price = bar._mem.low
             tick._mem.aggressor_side = AggressorSide.SELLER
-            trade_id_str = self._generate_trade_id_str()
-            tick._mem.trade_id = trade_id_new(pystr_to_cstr(trade_id_str))
+            tick._mem.trade_id = trade_id_new(pystr_to_cstr(self._generate_trade_id_str()))
             self._book.update_trade_tick(tick)
             self.iterate(tick.ts_init)
             self._core.set_last_raw(bar._mem.low.raw)
 
-        # Close
-        if bar._mem.close.raw != self._core.last_raw:  # Direct memory comparison
+    cdef void _process_trade_bar_close(self, Bar bar, TradeTick tick):
+        if bar._mem.close.raw != self._core.last_raw:
             if is_logging_initialized():
                 self._log.debug(f"Updating with close {bar.close}")
-            tick._mem.price = bar._mem.close  # Direct memory assignment
+            tick._mem.price = bar._mem.close
             tick._mem.aggressor_side = AggressorSide.BUYER if bar._mem.close.raw > self._core.last_raw else AggressorSide.SELLER
-            trade_id_str = self._generate_trade_id_str()
-            tick._mem.trade_id = trade_id_new(pystr_to_cstr(trade_id_str))
+            tick._mem.trade_id = trade_id_new(pystr_to_cstr(self._generate_trade_id_str()))
             self._book.update_trade_tick(tick)
             self.iterate(tick.ts_init)
             self._core.set_last_raw(bar._mem.close.raw)
@@ -705,8 +719,26 @@ cdef class OrderMatchingEngine:
         cdef Quantity bid_size = Quantity(self._last_bid_bar.volume.as_double() / 4.0, self._last_bid_bar._mem.volume.precision)
         cdef Quantity ask_size = Quantity(self._last_ask_bar.volume.as_double() / 4.0, self._last_ask_bar._mem.volume.precision)
 
-        # Create reusable tick
-        cdef QuoteTick tick = QuoteTick(
+        # Create base tick template
+        cdef QuoteTick tick = self._create_base_quote_tick(bid_size, ask_size)
+
+        # Process each price point
+        cdef bint process_high_first = not self._adaptive_bar_ordering or abs(self._last_bid_bar._mem.high.raw - self._last_bid_bar._mem.open.raw) < abs(self._last_bid_bar._mem.low.raw - self._last_bid_bar._mem.open.raw)
+
+        self._process_quote_bar_open(tick)
+        if process_high_first:
+            self._process_quote_bar_high(tick)
+            self._process_quote_bar_low(tick)
+        else:
+            self._process_quote_bar_low(tick)
+            self._process_quote_bar_high(tick)
+        self._process_quote_bar_close(tick)
+
+        self._last_bid_bar = None
+        self._last_ask_bar = None
+
+    cdef QuoteTick _create_base_quote_tick(self, Quantity bid_size, Quantity ask_size):
+        return QuoteTick(
             self._book.instrument_id,
             self._last_bid_bar.open,
             self._last_ask_bar.open,
@@ -716,32 +748,29 @@ cdef class OrderMatchingEngine:
             self._last_ask_bar.ts_init,
         )
 
-        # Open
+    cdef void _process_quote_bar_open(self, QuoteTick tick):
         self._book.update_quote_tick(tick)
         self.iterate(tick.ts_init)
 
-        # High
-        tick._mem.bid_price = self._last_bid_bar._mem.high  # Direct memory assignment
-        tick._mem.ask_price = self._last_ask_bar._mem.high  # Direct memory assignment
+    cdef void _process_quote_bar_high(self, QuoteTick tick):
+        tick._mem.bid_price = self._last_bid_bar._mem.high
+        tick._mem.ask_price = self._last_ask_bar._mem.high
         self._book.update_quote_tick(tick)
         self.iterate(tick.ts_init)
 
-        # Low
-        tick._mem.bid_price = self._last_bid_bar._mem.low  # Assigning memory directly
-        tick._mem.ask_price = self._last_ask_bar._mem.low  # Assigning memory directly
+    cdef void _process_quote_bar_low(self, QuoteTick tick):
+        tick._mem.bid_price = self._last_bid_bar._mem.low
+        tick._mem.ask_price = self._last_ask_bar._mem.low
         self._book.update_quote_tick(tick)
         self.iterate(tick.ts_init)
 
-        # Close
-        tick._mem.bid_price = self._last_bid_bar._mem.close  # Assigning memory directly
-        tick._mem.ask_price = self._last_ask_bar._mem.close  # Assigning memory directly
+    cdef void _process_quote_bar_close(self, QuoteTick tick):
+        tick._mem.bid_price = self._last_bid_bar._mem.close
+        tick._mem.ask_price = self._last_ask_bar._mem.close
         self._book.update_quote_tick(tick)
         self.iterate(tick.ts_init)
 
-        self._last_bid_bar = None
-        self._last_ask_bar = None
-
-# -- TRADING COMMANDS -----------------------------------------------------------------------------
+    # -- TRADING COMMANDS -----------------------------------------------------------------------------
 
     cpdef void process_order(self, Order order, AccountId account_id):
         if self._core.order_exists(order.client_order_id):

nautilus_trader/backtest/node.py

@@ -271,6 +271,7 @@ def _create_engine(
                 use_position_ids=config.use_position_ids,
                 use_random_ids=config.use_random_ids,
                 use_reduce_only=config.use_reduce_only,
+                adaptive_bar_ordering=config.adaptive_bar_ordering,
             )
 
         # Add instruments

nautilus_trader/data/aggregation.pxd

@@ -13,7 +13,6 @@
 #  limitations under the License.
 # -------------------------------------------------------------------------------------------------
 
-from cpython.datetime cimport datetime
 from cpython.datetime cimport timedelta
 from libc.stdint cimport uint8_t
 from libc.stdint cimport uint64_t